From 73565460729138f6ec30f74eef4fbf361d370886 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Thu, 9 Nov 2017 18:35:12 -0500 Subject: [PATCH 1/3] Add benchmark prototype Build a page with webpack that loads a project and automatically runs it, collecting performance information for a set amount of time, stopping and displaying the results. --- src/benchmark/benchmark.css | 7 + src/benchmark/benchmark.js | 460 ++++++++++++++++++++++++++++++++++++ src/benchmark/index.html | 69 ++++++ webpack.config.js | 63 +++++ 4 files changed, 599 insertions(+) create mode 100644 src/benchmark/benchmark.css create mode 100644 src/benchmark/benchmark.js create mode 100644 src/benchmark/index.html diff --git a/src/benchmark/benchmark.css b/src/benchmark/benchmark.css new file mode 100644 index 000000000..e843b5e94 --- /dev/null +++ b/src/benchmark/benchmark.css @@ -0,0 +1,7 @@ +@media (min-width: 960px) { + .profile-tables { + position: absolute; + top: 0; + left: 520px; + } +} diff --git a/src/benchmark/benchmark.js b/src/benchmark/benchmark.js new file mode 100644 index 000000000..d8540404a --- /dev/null +++ b/src/benchmark/benchmark.js @@ -0,0 +1,460 @@ +const Scratch = window.Scratch = window.Scratch || {}; + +const ASSET_SERVER = 'https://cdn.assets.scratch.mit.edu/'; +const PROJECT_SERVER = 'https://cdn.projects.scratch.mit.edu/'; + +const SLOW = .1; + +const loadProject = function () { + let id = location.hash.substring(1); + if (id.length < 1 || !isFinite(id)) { + id = '119615668'; + } + Scratch.vm.downloadProjectId(id); +}; + +/** + * @param {Asset} asset - calculate a URL for this asset. + * @returns {string} a URL to download a project file. + */ +const getProjectUrl = function (asset) { + const assetIdParts = asset.assetId.split('.'); + const assetUrlParts = [PROJECT_SERVER, 'internalapi/project/', assetIdParts[0], '/get/']; + if (assetIdParts[1]) { + assetUrlParts.push(assetIdParts[1]); + } + return assetUrlParts.join(''); +}; + +/** + * @param {Asset} asset - calculate a URL for this asset. + * @returns {string} a URL to download a project asset (PNG, WAV, etc.) + */ +const getAssetUrl = function (asset) { + const assetUrlParts = [ + ASSET_SERVER, + 'internalapi/asset/', + asset.assetId, + '.', + asset.dataFormat, + '/get/' + ]; + return assetUrlParts.join(''); +}; + +class LoadingProgress { + constructor (callback) { + this.total = 0; + this.complete = 0; + this.callback = callback; + } + + on (storage) { + const _this = this; + const _load = storage.webHelper.load; + storage.webHelper.load = function (...args) { + const result = _load.call(this, ...args); + _this.total += 1; + _this.callback(_this); + result.then(() => { + _this.complete += 1; + _this.callback(_this); + }); + return result; + }; + } +} + +class StatTable { + constructor ({table, keys, viewOf, isSlow}) { + this.table = table; + if (keys) { + this.keys = keys; + } + if (viewOf) { + this.viewOf = viewOf; + } + if (isSlow) { + this.isSlow = isSlow; + } + } + + render () { + const table = this.table; + Array.from(table.children) + .forEach(node => table.removeChild(node)); + const keys = this.keys(); + for (const key of keys) { + this.viewOf(key).render({ + table, + isSlow: frame => this.isSlow(key, frame) + }); + } + } +} + +class StatView { + constructor (name) { + this.name = name; + this.executions = 0; + this.selfTime = 0; + this.totalTime = 0; + } + + update (selfTime, totalTime) { + this.executions++; + this.selfTime += selfTime; + this.totalTime += totalTime; + } + + render ({table, isSlow}) { + const row = document.createElement('tr'); + let cell = document.createElement('td'); + cell.innerText = this.name; + row.appendChild(cell); + + if (isSlow(this)) { + row.setAttribute('class', 'slow'); + } + + cell = document.createElement('td'); + cell.innerText = (this.selfTime / 1000).toPrecision(3); + row.appendChild(cell); + + cell = document.createElement('td'); + cell.innerText = (this.totalTime / 1000).toPrecision(3); + row.appendChild(cell); + + cell = document.createElement('td'); + cell.innerText = this.executions; + row.appendChild(cell); + + table.appendChild(row); + } +} + +class RunningStats { + constructor (profiler) { + this.stepThreadsInnerId = profiler.idByName('Sequencer.stepThreads#inner'); + this.blockFunctionId = profiler.idByName('blockFunction'); + this.stpeThreadsId = profiler.idByName('Sequencer.stepThreads'); + + this.recordedTime = 0; + this.executed = { + steps: 0, + blocks: 0 + }; + } + + update (id, selfTime, totalTime) { + if (id === this.stpeThreadsId) { + this.recordedTime += totalTime; + } else if (id === this.stepThreadsInnerId) { + this.executed.steps++; + } else if (id === this.blockFunctionId) { + this.executed.blocks++; + } + } +} + +const WORK_TIME = 0.75; + +class RunningStatsView { + constructor ({runningStats, maxRecordedTime, dom}) { + this.recordedTimeDom = + dom.getElementsByClassName('profile-count-amount-recorded')[0]; + this.stepsLoopedDom = + dom.getElementsByClassName('profile-count-steps-looped')[0]; + this.blocksExecutedDom = + dom.getElementsByClassName('profile-count-blocks-executed')[0]; + + this.maxRecordedTime = maxRecordedTime; + this.maxWorkedTime = maxRecordedTime * WORK_TIME; + this.runningStats = runningStats; + } + + render () { + const { + runningStats, + recordedTimeDom, + stepsLoopedDom, + blocksExecutedDom + } = this; + const {executed} = runningStats; + const fractionWorked = runningStats.recordedTime / this.maxWorkedTime; + recordedTimeDom.innerText = `${(fractionWorked * 100).toFixed(1)} %`; + stepsLoopedDom.innerText = executed.steps; + blocksExecutedDom.innerText = executed.blocks; + } +} + +class Frames { + constructor (profiler) { + this.profiler = profiler; + + this.frames = []; + } + + update (id, selfTime, totalTime) { + if (!this.frames[id]) { + this.frames[id] = new StatView(this.profiler.nameById(id)); + } + this.frames[id].update(selfTime, totalTime); + } +} + +const frameOrder = [ + 'blockFunction', + 'execute', + 'Sequencer.stepThread', + 'Sequencer.stepThreads#inner', + 'Sequencer.stepThreads', + 'RenderWebGL.draw', + 'Runtime._step' +]; + +class FramesTable extends StatTable { + constructor (options) { + super(options); + + this.profiler = options.profiler; + this.frames = options.frames; + } + + keys () { + const keys = Object.keys(this.frames.frames) + .map(id => this.profiler.nameById(Number(id))); + keys.sort((a, b) => frameOrder.indexOf(a) - frameOrder.indexOf(b)); + return keys; + } + + viewOf (key) { + return this.frames.frames[this.profiler.idByName(key)]; + } + + isSlow (key, frame) { + return ([ + 'Sequencer.stepThreads', + 'Sequencer.stepThreads#inner', + 'Sequencer.stepThread', + 'execute' + ].indexOf(key) > 0 && + frame.selfTime / frame.totalTime > SLOW); + } +} + +class Opcodes { + constructor (profiler) { + this.blockFunctionId = profiler.idByName('blockFunction'); + + this.opcodes = {}; + } + + update (id, selfTime, totalTime, arg) { + if (id === this.blockFunctionId) { + if (!this.opcodes[arg]) { + this.opcodes[arg] = new StatView(arg); + } + this.opcodes[arg].update(selfTime, totalTime); + } + } +} + +class OpcodeTable extends StatTable { + constructor (options) { + super(options); + + this.profiler = options.profiler; + this.opcodes = options.opcodes; + this.frames = options.frames; + } + + keys () { + const keys = Object.keys(this.opcodes.opcodes); + keys.sort(); + return keys; + } + + viewOf (key) { + return this.opcodes.opcodes[key]; + } + + isSlow (key) { + const blockFunctionTotalTime = this.frames.frames[this.profiler.idByName('blockFunction')].totalTime; + const rowTotalTime = this.opcodes.opcodes[key].totalTime; + const percentOfRun = rowTotalTime / blockFunctionTotalTime; + return percentOfRun > SLOW; + } +} + +class ProfilerRun { + constructor ({vm, maxRecordedTime, warmUpTime}) { + this.vm = vm; + this.maxRecordedTime = maxRecordedTime; + this.warmUpTime = warmUpTime; + + vm.runtime.enableProfiling(); + const profiler = this.profiler = vm.runtime.profiler; + vm.runtime.profiler = null; + + const runningStats = this.runningStats = new RunningStats(profiler); + const runningStatsView = this.runningStatsView = new RunningStatsView({ + dom: document.getElementsByClassName('profile-count-group')[0], + + runningStats, + maxRecordedTime: 6000 + }); + + const frames = this.frames = new Frames(profiler); + this.frameTable = new FramesTable({ + table: document + .getElementsByClassName('profile-count-frame-table')[0] + .getElementsByTagName('tbody')[0], + + profiler, + frames + }); + + const opcodes = this.opcodes = new Opcodes(profiler); + this.opcodeTable = new OpcodeTable({ + table: document + .getElementsByClassName('profile-count-opcode-table')[0] + .getElementsByTagName('tbody')[0], + + profiler, + opcodes, + frames + }); + + const stepId = profiler.idByName('Runtime._step'); + profiler.onFrame = ({id, selfTime, totalTime, arg}) => { + if (id === stepId) { + runningStatsView.render(); + } + runningStats.update(id, selfTime, totalTime, arg); + opcodes.update(id, selfTime, totalTime, arg); + frames.update(id, selfTime, totalTime, arg); + }; + } + + run () { + loadProject(); + + this.vm.on('workspaceUpdate', () => { + setTimeout(() => { + this.vm.greenFlag(); + }, 100); + setTimeout(() => { + this.vm.runtime.profiler = this.profiler; + }, 100 + this.warmUpTime); + setTimeout(() => { + this.vm.stopAll(); + clearTimeout(this.vm.runtime._steppingInterval); + this.vm.runtime.profiler = null; + + this.frameTable.render(); + this.opcodeTable.render(); + }, 100 + this.warmUpTime + this.maxRecordedTime); + }); + } +} + +window.onload = function () { + // Lots of global variables to make debugging easier + // Instantiate the VM. + const vm = new window.VirtualMachine(); + Scratch.vm = vm; + + vm.setTurboMode(true); + + const storage = new ScratchStorage(); /* global ScratchStorage */ + const AssetType = storage.AssetType; + storage.addWebSource([AssetType.Project], getProjectUrl); + storage.addWebSource([AssetType.ImageVector, AssetType.ImageBitmap, AssetType.Sound], getAssetUrl); + vm.attachStorage(storage); + + new LoadingProgress(progress => { + document.getElementsByClassName('loading-total')[0] + .innerText = progress.total; + document.getElementsByClassName('loading-complete')[0] + .innerText = progress.complete; + }).on(storage); + + new ProfilerRun({ + vm, + warmUpTime: 4000, + maxRecordedTime: 6000 + }).run(); + + // Instantiate the renderer and connect it to the VM. + const canvas = document.getElementById('scratch-stage'); + const renderer = new window.RenderWebGL(canvas); + Scratch.renderer = renderer; + vm.attachRenderer(renderer); + const audioEngine = new window.AudioEngine(); + vm.attachAudioEngine(audioEngine); + + // Feed mouse events as VM I/O events. + document.addEventListener('mousemove', e => { + const rect = canvas.getBoundingClientRect(); + const coordinates = { + x: e.clientX - rect.left, + y: e.clientY - rect.top, + canvasWidth: rect.width, + canvasHeight: rect.height + }; + Scratch.vm.postIOData('mouse', coordinates); + }); + canvas.addEventListener('mousedown', e => { + const rect = canvas.getBoundingClientRect(); + const data = { + isDown: true, + x: e.clientX - rect.left, + y: e.clientY - rect.top, + canvasWidth: rect.width, + canvasHeight: rect.height + }; + Scratch.vm.postIOData('mouse', data); + e.preventDefault(); + }); + canvas.addEventListener('mouseup', e => { + const rect = canvas.getBoundingClientRect(); + const data = { + isDown: false, + x: e.clientX - rect.left, + y: e.clientY - rect.top, + canvasWidth: rect.width, + canvasHeight: rect.height + }; + Scratch.vm.postIOData('mouse', data); + e.preventDefault(); + }); + + // Feed keyboard events as VM I/O events. + document.addEventListener('keydown', e => { + // Don't capture keys intended for Blockly inputs. + if (e.target !== document && e.target !== document.body) { + return; + } + Scratch.vm.postIOData('keyboard', { + keyCode: e.keyCode, + isDown: true + }); + e.preventDefault(); + }); + document.addEventListener('keyup', e => { + // Always capture up events, + // even those that have switched to other targets. + Scratch.vm.postIOData('keyboard', { + keyCode: e.keyCode, + isDown: false + }); + // E.g., prevent scroll. + if (e.target !== document && e.target !== document.body) { + e.preventDefault(); + } + }); + + // Run threads + vm.start(); +}; diff --git a/src/benchmark/index.html b/src/benchmark/index.html new file mode 100644 index 000000000..7ddc98dc3 --- /dev/null +++ b/src/benchmark/index.html @@ -0,0 +1,69 @@ + + + + + + Scratch VM Benchmark + + + + +

Scratch VM Benchmark

+
+
+ + 0 / 0 +
+
+
+ + ... +
+
+ + ... +
+
+ + ... +
+
+ +
+ + + + + + + +
FrameSelf TimeTotal TimeExecutions
+ + + + + + + + + + +
opcodeSelf TimeTotal TimeExecutions
+
+ +
+ + + + + + + + + + + diff --git a/webpack.config.js b/webpack.config.js index f2d4c880c..6b32c6d83 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -130,5 +130,68 @@ module.exports = [ from: 'src/playground' }]) ]) + }), + // Benchmark + defaultsDeep({}, base, { + target: 'web', + entry: { + 'scratch-vm': './src/index.js', + 'vendor': [ + // FPS counter + 'stats.js/build/stats.min.js', + // Syntax highlighter + 'highlightjs/highlight.pack.min.js', + // Scratch Blocks + 'scratch-blocks/dist/vertical.js', + // Audio + 'scratch-audio', + // Renderer + 'scratch-render' + ] + }, + output: { + path: path.resolve(__dirname, 'benchmark'), + filename: '[name].js' + }, + module: { + rules: base.module.rules.concat([ + { + test: require.resolve('./src/index.js'), + loader: 'expose-loader?VirtualMachine' + }, + { + test: require.resolve('stats.js/build/stats.min.js'), + loader: 'script-loader' + }, + { + test: require.resolve('highlightjs/highlight.pack.min.js'), + loader: 'script-loader' + }, + { + test: require.resolve('scratch-blocks/dist/vertical.js'), + loader: 'expose-loader?Blockly' + }, + { + test: require.resolve('scratch-audio'), + loader: 'expose-loader?AudioEngine' + }, + { + test: require.resolve('scratch-render'), + loader: 'expose-loader?RenderWebGL' + } + ]) + }, + plugins: base.plugins.concat([ + new CopyWebpackPlugin([{ + from: 'node_modules/scratch-blocks/media', + to: 'media' + }, { + from: 'node_modules/highlightjs/styles/zenburn.css' + }, { + from: 'node_modules/scratch-storage/dist/web' + }, { + from: 'src/benchmark' + }]) + ]) }) ]; From a7314d6f0d6e9078a26051ccefe58ff814bb67de Mon Sep 17 00:00:00 2001 From: Boaz Sender Date: Thu, 16 Nov 2017 15:43:51 -0500 Subject: [PATCH 2/3] adds ui styling and context to benchmark prototype --- src/benchmark/benchmark.css | 54 +++++++++++++++++++++++++++++++++++-- src/benchmark/benchmark.js | 10 ++++++- src/benchmark/index.html | 26 +++++++++++++----- 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/src/benchmark/benchmark.css b/src/benchmark/benchmark.css index e843b5e94..07068f59b 100644 --- a/src/benchmark/benchmark.css +++ b/src/benchmark/benchmark.css @@ -1,7 +1,57 @@ +body { + font-family: monospace; +} +p { + max-width: 400px; +} @media (min-width: 960px) { .profile-tables { + top: 0px; position: absolute; - top: 0; - left: 520px; + left: 450px; } } +#scratch-stage { + border: 5px solid black; + display: block; + width: 400px; + height: 300px; +} +.loading label, .profile-count label{ + width: 15em; + display: inline-block; +} +.profile-tables table { + margin: 30px 0 30px 0px; +} +.profile-tables th { + border-bottom: 1px solid #333; + text-align: center; +} +.profile-tables th:first-child { + width: 215px; +} +.profile-tables th, .profile-tables td { + min-width: 85px; + border-bottom: 1px solid #ccc; + border-spacing: 0; + border-collapse: collapse; + padding: 5px; +} +.profile-tables td:not(:first-child) { + text-align: center; +} +.profile-tables img{ + margin: 0 auto; + display: block; + clear: both; + width: 20%; +} + +.slow { + background-color: #ffa1a1; +} +.profiler-count-running { + height: 4em; + background-color: #dddddd; +} diff --git a/src/benchmark/benchmark.js b/src/benchmark/benchmark.js index d8540404a..3fd1bc2a2 100644 --- a/src/benchmark/benchmark.js +++ b/src/benchmark/benchmark.js @@ -5,10 +5,18 @@ const PROJECT_SERVER = 'https://cdn.projects.scratch.mit.edu/'; const SLOW = .1; +const projectInput = document.querySelector('input'); + +document.querySelector('.run') + .addEventListener('click', () => { + window.location.hash = projectInput.value; + location.reload(); + }, false); + const loadProject = function () { let id = location.hash.substring(1); if (id.length < 1 || !isFinite(id)) { - id = '119615668'; + id = projectInput.value; } Scratch.vm.downloadProjectId(id); }; diff --git a/src/benchmark/index.html b/src/benchmark/index.html index 7ddc98dc3..9e6b885a4 100644 --- a/src/benchmark/index.html +++ b/src/benchmark/index.html @@ -4,16 +4,28 @@ Scratch VM Benchmark - -

Scratch VM Benchmark

-
+

Scratch VM Benchmark

+

+ Welcome to the scratch-vm benchmark. This tool helps you profile a scratch + project. When you load the page, it: +

    +
  1. loads the default project +
  2. runs the project for 4 seconds to warm up +
  3. profiles for 6 seconds +
  4. stops and reports +
+

+ + +

+ Try a different project, like `130041250` +

+ +
+
0 / 0 From aa0acc62e4ba9c379af496f6ae7abadbf6bebe85 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 20 Nov 2017 17:45:55 -0500 Subject: [PATCH 3/3] Add benchmark to ignore files --- .eslintignore | 1 + .gitignore | 1 + 2 files changed, 2 insertions(+) diff --git a/.eslintignore b/.eslintignore index 8af9aff65..f571fb2a3 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2,3 +2,4 @@ coverage/* dist/* node_modules/* playground/* +benchmark/* diff --git a/.gitignore b/.gitignore index 2f3825877..97bbbfa69 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ npm-* # Build /dist /playground +/benchmark