From b012d60689e41a04de701625495b3e001835e6ae Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Tue, 27 Mar 2018 18:09:19 -0400 Subject: [PATCH 1/9] Add video sensing debug C_CELL and AB_CELL views - Rename UV to UV_CELL, UV cannot be rendered in a non-cell format Modify C and AB views as well to better represent in color the work being done. If you watch the debug views with XY_CELL, AB_CELL, C_CELL, and UV_CELL you can see how AB_CELL and C_CELL affect UV_CELL in similar color patterns. --- src/extensions/scratch3_video_sensing/view.js | 88 ++++++++++++++++--- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/view.js b/src/extensions/scratch3_video_sensing/view.js index 665a9acfa..9a50c8ea4 100644 --- a/src/extensions/scratch3_video_sensing/view.js +++ b/src/extensions/scratch3_video_sensing/view.js @@ -6,15 +6,17 @@ const THRESHOLD = 10; const OUTPUT = { INPUT: -1, - XYT: 0, - XYT_CELL: 1, - XY: 2, - XY_CELL: 3, + XY: 0, + XY_CELL: 1, + AB: 2, + AB_CELL: 3, T: 4, T_CELL: 5, - C: 6, - AB: 7, - UV: 8 + XYT: 6, + XYT_CELL: 7, + C: 8, + C_CELL: 9, + UV_CELL: 10 }; class VideoMotionView { @@ -186,19 +188,79 @@ class VideoMotionView { const {gradX, gradY, gradT} = this._grads(address); buffer[address] = (0xff << 24) + - ((gradY * gradT) << 8) + - (gradX * gradT); + (((Math.sqrt(gradY * gradT) * 0x0f) & 0xff) << 8) + + ((Math.sqrt(gradX * gradT) * 0x0f) & 0xff); + }); + } + if (this.output === OUTPUT.C_CELL) { + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { + let C2 = 0; + let C1 = 0; + let n = 0; + + eachAddress(address => { + const {gradX, gradY, gradT} = this._grads(address); + C2 += gradX * gradT; + C1 += gradY * gradT; + n += 1; + }); + + C2 = Math.sqrt(C2); + C1 = Math.sqrt(C1); + + eachAddress(address => { + buffer[address] = + (0xff << 24) + + ((C1 & 0xff) << 8) + + ((C2 & 0xff) << 0); + }); }); } else if (this.output === OUTPUT.AB) { this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { const {gradX, gradY} = this._grads(address); buffer[address] = (0xff << 24) + - ((gradX * gradY) << 16) + - ((gradY * gradY) << 8) + - (gradX * gradX); + (((gradX * gradY) & 0xff) << 16) + + (((gradY * gradY) & 0xff) << 8) + + ((gradX * gradX) & 0xff); }); - } else if (this.output === OUTPUT.UV) { + } + if (this.output === OUTPUT.AB_CELL) { + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let n = 0; + + eachAddress(address => { + const {gradX, gradY} = this._grads(address); + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + n += 1; + }); + + A2 = Math.sqrt(A2); + A1B2 = Math.sqrt(A1B2); + B1 = Math.sqrt(B1); + + eachAddress(address => { + buffer[address] = + (0xff << 24) + + ((A1B2 & 0xff) << 16) + + ((B1 & 0xff) << 8) + + (A2 & 0xff); + }); + }); + } else if (this.output === OUTPUT.UV_CELL) { const winStep = (WINSIZE * 2) + 1; const wmax = WIDTH - WINSIZE - 1; const hmax = HEIGHT - WINSIZE - 1; From cfd74a7c7f2001aacb02caf96d531255f32465bb Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Tue, 27 Mar 2018 18:11:46 -0400 Subject: [PATCH 2/9] Add checkboxes for each video sensing debug view --- src/playground/motion.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/playground/motion.js b/src/playground/motion.js index 8d801d755..a7db4796a 100644 --- a/src/playground/motion.js +++ b/src/playground/motion.js @@ -36,7 +36,7 @@ .map(output => new VideoMotionView(motion, output)); const view = views[0]; - const defaultViews = [OUTPUT.INPUT, OUTPUT.XY_CELL, OUTPUT.T_CELL, OUTPUT.UV]; + const defaultViews = [OUTPUT.INPUT, OUTPUT.XY_CELL, OUTPUT.T_CELL, OUTPUT.UV_CELL]; const activators = document.createElement('div'); activators.style.userSelect = 'none'; From e65b7985cb040c0c29d4dc000b2432bd6547a388 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:48:07 -0400 Subject: [PATCH 3/9] Add whenMotionGreaterThan hat block to video sensing ext --- .../scratch3_video_sensing/index.js | 79 +++++++++++-------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 506d114f5..026588083 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -272,6 +272,19 @@ class Scratch3VideoSensingBlocks { defaultValue: 1 } } + }, + { + // @todo this hat needs to be set itself to restart existing + // threads like Scratch 2's behaviour. + opcode: 'whenMotionGreaterThan', + text: 'when video motion > [REFERENCE]', + blockType: BlockType.HAT, + arguments: { + REFERENCE: { + type: ArgumentType.NUMBER, + defaultValue: 10 + } + } } ], menus: { @@ -281,14 +294,32 @@ class Scratch3VideoSensingBlocks { }; } + /** + * Analyze a part of the frame that a target overlaps. + * @param {Target} target - a target to determine where to analyze + * @returns {MotionState} the motion state for the given target + */ + _analyzeLocalMotion (target) { + const drawable = this.runtime.renderer._allDrawables[target.drawableID]; + const state = this._getMotionState(target); + this.detect.getLocalMotion(drawable, state); + return state; + } + + /** + * A scratch reporter block handle that analyzes the last two frames and + * depending on the arguments, returns the motion or direction for the + * whole stage or just the target sprite. + * @param {object} args - the block arguments + * @param {BlockUtility} util - the block utility + * @returns {number} the motion amount or direction of the stage or sprite + */ videoOn (args, util) { this.detect.analyzeFrame(); let state = this.detect; if (Number(args.STAGE_SPRITE) === 2) { - const drawable = this.runtime.renderer._allDrawables[util.target.drawableID]; - state = this._getMotionState(util.target); - this.detect.getLocalMotion(drawable, state); + state = this._analyzeLocalMotion(util.target); } if (Number(args.MOTION_DIRECTION) === 1) { @@ -298,38 +329,18 @@ class Scratch3VideoSensingBlocks { } /** - * Check if the stack timer needs initialization. - * @param {object} util - utility object provided by the runtime. - * @return {boolean} - true if the stack timer needs to be initialized. - * @private + * A scratch hat block edge handle that analyzes the last two frames where + * the target sprite overlaps and if it has more motion than the given + * reference value. + * @param {object} args - the block arguments + * @param {BlockUtility} util - the block utility + * @returns {boolean} true if the sprite overlaps more motion than the + * reference */ - _stackTimerNeedsInit (util) { - return !util.stackFrame.timer; - } - - /** - * Start the stack timer and the yield the thread if necessary. - * @param {object} util - utility object provided by the runtime. - * @param {number} duration - a duration in seconds to set the timer for. - * @private - */ - _startStackTimer (util, duration) { - util.stackFrame.timer = new Timer(); - util.stackFrame.timer.start(); - util.stackFrame.duration = duration; - util.yield(); - } - - /** - * Check the stack timer, and if its time is not up yet, yield the thread. - * @param {object} util - utility object provided by the runtime. - * @private - */ - _checkStackTimer (util) { - const timeElapsed = util.stackFrame.timer.timeElapsed(); - if (timeElapsed < util.stackFrame.duration * 1000) { - util.yield(); - } + whenMotionGreaterThan (args, util) { + this.detect.analyzeFrame(); + const state = this._analyzeLocalMotion(util.target); + return state.motionAmount > Number(args.REFERENCE); } } From c291a04cc7d7a816be702fbced432ac036e29365 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:49:00 -0400 Subject: [PATCH 4/9] Add file comment block to video_sensing/debug.js --- src/extensions/scratch3_video_sensing/debug.js | 6 ++++++ src/extensions/scratch3_video_sensing/index.js | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/extensions/scratch3_video_sensing/debug.js b/src/extensions/scratch3_video_sensing/debug.js index a7a304880..24603b584 100644 --- a/src/extensions/scratch3_video_sensing/debug.js +++ b/src/extensions/scratch3_video_sensing/debug.js @@ -1,3 +1,9 @@ +/** + * A debug "index" module exporting VideoMotion and VideoMotionView to debug + * VideoMotion directly. + * @file debug.js + */ + const VideoMotion = require('./lib'); const VideoMotionView = require('./view'); diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 026588083..2904f5f54 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -2,7 +2,6 @@ const ArgumentType = require('../../extension-support/argument-type'); const BlockType = require('../../extension-support/block-type'); const Clone = require('../../util/clone'); const log = require('../../util/log'); -const Timer = require('../../util/timer'); const VideoMotion = require('./lib'); From b46c956203cbb501359c2ef65ba5b2c720d539a3 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:50:53 -0400 Subject: [PATCH 5/9] Comment items in Scratch3VideoSensingBlocks --- .../scratch3_video_sensing/index.js | 154 ++++++++++++++---- 1 file changed, 125 insertions(+), 29 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 2904f5f54..10658f0c6 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -1,3 +1,5 @@ +const Runtime = require('../../engine/runtime'); + const ArgumentType = require('../../extension-support/argument-type'); const BlockType = require('../../extension-support/block-type'); const Clone = require('../../util/clone'); @@ -32,32 +34,123 @@ class Scratch3VideoSensingBlocks { */ this.runtime = runtime; + /** + * The motion detection algoritm used to power the motion amount and + * direction values. + * @type {VideoMotion} + */ this.detect = new VideoMotion(); + /** + * The last millisecond epoch timestamp that the video stream was + * analyzed. + * @type {number} + */ this._lastUpdate = null; + /** + * Id representing a Scratch Renderer skin the video is rendered to for + * previewing. + * @type {number} + */ this._skinId = -1; + + /** + * The Scratch Renderer Skin object. + * @type {Skin} + */ this._skin = null; + + /** + * Id for a drawable using the video's skin that will render as a video + * preview. + * @type {Drawable} + */ this._drawable = -1; + /** + * Canvas DOM element video is rendered to down or up sample to the + * expected resolution. + * @type {HTMLCanvasElement} + */ + this._sampleCanvas = null; + + /** + * Canvas 2D Context to render to the _sampleCanvas member. + * @type {CanvasRenderingContext2D} + */ + this._sampleContext = null; + + // Clear target motion state values when the project starts. + this.runtime.on(Runtime.PROJECT_RUN_START, this.reset.bind(this)); + + // Boot up the video, canvas to down/up sample the video stream, the + // preview skin and drawable, and kick off looping the analysis logic. this._setupVideo(); this._setupSampleCanvas(); this._setupPreview(); this._loop(); } - static get INTERVAL () { - return 33; - } - + /** + * Dimensions the video stream is analyzed at after its rendered to the + * sample canvas. + * @type {Array.} + */ static get DIMENSIONS () { return [480, 360]; } + /** + * Order preview drawable is inserted at in the renderer. + * @type {number} + */ static get ORDER () { return 1; } + /** + * The key to load & store a target's motion-related state. + * @type {string} + */ + static get STATE_KEY () { + return 'Scratch.videoSensing'; + } + + /** + * The default motion-related state, to be used when a target has no existing motion state. + * @type {MotionState} + */ + static get DEFAULT_MOTION_STATE () { + return { + motionFrameNumber: 0, + motionAmount: 0, + motionDirection: 0 + }; + } + + /** + * Reset the extension's data motion detection data. This will clear out + * for example old frames, so the first analyzed frame will not be compared + * against a frame from before reset was called. + */ + reset () { + this.detect.reset(); + + const targets = this.runtime.targets; + for (let i = 0; i < targets.length; i++) { + const state = targets[i].getCustomState(Scratch3VideoSensingBlocks.STATE_KEY); + if (state) { + state.motionAmount = 0; + state.motionDirection = 0; + } + } + } + + /** + * Setup a video element connected to a user media stream. + * @private + */ _setupVideo () { this._video = document.createElement('video'); navigator.getUserMedia({ @@ -80,6 +173,11 @@ class Scratch3VideoSensingBlocks { }); } + /** + * Create a campus to render the user media video to down/up sample to the + * needed resolution. + * @private + */ _setupSampleCanvas () { // Create low-resolution image to sample video for analysis and preview const canvas = this._sampleCanvas = document.createElement('canvas'); @@ -88,6 +186,11 @@ class Scratch3VideoSensingBlocks { this._sampleContext = canvas.getContext('2d'); } + /** + * Create a Scratch Renderer Skin and Drawable to preview the user media + * video stream. + * @private + */ _setupPreview () { if (this._skinId !== -1) return; if (this._skin !== null) return; @@ -106,6 +209,11 @@ class Scratch3VideoSensingBlocks { }); } + /** + * Occasionally step a loop to sample the video, stamp it to the preview + * skin, and add a TypedArray copy of the canvas's pixel data. + * @private + */ _loop () { setTimeout(this._loop.bind(this), this.runtime.currentStepTime); @@ -164,9 +272,11 @@ class Scratch3VideoSensingBlocks { } /** - * Create data for a menu in scratch-blocks format, consisting of an array of objects with text and - * value properties. The text is a translated string, and the value is one-indexed. - * @param {object[]} info - An array of info objects each having a name property. + * Create data for a menu in scratch-blocks format, consisting of an array + * of objects with text and value properties. The text is a translated + * string, and the value is one-indexed. + * @param {object[]} info - An array of info objects each having a name + * property. * @return {array} - An array of objects with text and value properties. * @private */ @@ -179,27 +289,10 @@ class Scratch3VideoSensingBlocks { }); } - /** - * The key to load & store a target's motion-related state. - * @type {string} - */ - static get STATE_KEY () { - return 'Scratch.videoSensing'; - } - - /** - * The default music-related state, to be used when a target has no existing music state. - * @type {MusicState} - */ - static get DEFAULT_MOTION_STATE () { - return { - currentInstrument: 0 - }; - } - /** * @param {Target} target - collect motion state for this target. - * @returns {MotionState} the mutable motion state associated with that target. This will be created if necessary. + * @returns {MotionState} the mutable motion state associated with that + * target. This will be created if necessary. * @private */ _getMotionState (target) { @@ -212,7 +305,8 @@ class Scratch3VideoSensingBlocks { } /** - * An array of info about each drum. + * An array of choices of whether a reporter should return the frame's + * motion amount or direction. * @type {object[]} an array of objects. * @param {string} name - the translatable name to display in the drums menu. * @param {string} fileName - the name of the audio file containing the drum sound. @@ -231,8 +325,10 @@ class Scratch3VideoSensingBlocks { /** * An array of info about each drum. * @type {object[]} an array of objects. - * @param {string} name - the translatable name to display in the drums menu. - * @param {string} fileName - the name of the audio file containing the drum sound. + * @param {string} name - the translatable name to display in the drums + * menu. + * @param {string} fileName - the name of the audio file containing the + * drum sound. */ get STAGE_SPRITE_INFO () { return [ From a32b15a8f9a93712d4d8aba22d8f687cf6295e68 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:51:47 -0400 Subject: [PATCH 6/9] Add VideoSensing math module - motionVector takes motion component and returns a 2d vector - scratchAtan2 takes a y and x value and returns an angle in degrees referencing Scratch's coordinate system --- src/extensions/scratch3_video_sensing/math.js | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/extensions/scratch3_video_sensing/math.js diff --git a/src/extensions/scratch3_video_sensing/math.js b/src/extensions/scratch3_video_sensing/math.js new file mode 100644 index 000000000..b6357b1f1 --- /dev/null +++ b/src/extensions/scratch3_video_sensing/math.js @@ -0,0 +1,76 @@ +/** + * A constant value helping to transform a value in radians to degrees. + * @type {number} + */ +const TO_DEGREE = 180 / Math.PI; + +/** + * A object reused to save on memory allocation returning u and v vector from + * motionVector. + * @type {UV} + */ +const _motionVectorOut = {u: 0, v: 0}; + +/** + * Determine a motion vector combinations of the color component difference on + * the x axis, y axis, and temporal axis. + * @param {number} A2 - a sum of x axis squared + * @param {number} A1B2 - a sum of x axis times y axis + * @param {number} B1 - a sum of y axis squared + * @param {number} C2 - a sum of x axis times temporal axis + * @param {number} C1 - a sum of y axis times temporal axis + * @param {UV} out - optional object to store return UV info in + * @returns {UV} a uv vector representing the motion for the given input + */ +const motionVector = function (A2, A1B2, B1, C2, C1, out = _motionVectorOut) { + // Compare sums of X * Y and sums of X squared and Y squared. + const delta = ((A1B2 * A1B2) - (A2 * B1)); + if (delta) { + // System is not singular - solving by Kramer method. + const deltaX = -((C1 * A1B2) - (C2 * B1)); + const deltaY = -((A1B2 * C2) - (A2 * C1)); + const Idelta = 8 / delta; + out.u = deltaX * Idelta; + out.v = deltaY * Idelta; + } else { + // Singular system - find optical flow in gradient direction. + const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); + if (Norm) { + const IGradNorm = 8 / Norm; + const temp = -(C1 + C2) * IGradNorm; + out.u = (A1B2 + A2) * temp; + out.v = (B1 + A1B2) * temp; + } else { + out.u = 0; + out.v = 0; + } + } + return out; +}; + +/** + * Translate an angle in degrees with the range -180 to 180 rotated to + * Scratch's reference angle. + * @param {number} degrees - angle in range -180 to 180 + * @returns {number} angle from Scratch's reference angle + */ +const scratchDegrees = function (degrees) { + return ((degrees + 270) % 360) - 180; +}; + +/** + * Get the angle of the y and x component of a 2d vector in degrees in + * Scratch's coordinate plane. + * @param {number} y - the y component of a 2d vector + * @param {number} x - the x component of a 2d vector + * @returns {number} angle in degrees in Scratch's coordinate plane + */ +const scratchAtan2 = function (y, x) { + return scratchDegrees(Math.atan2(y, x) * TO_DEGREE); +}; + +module.exports = { + motionVector, + scratchDegrees, + scratchAtan2 +}; From 9a2e9372711da7fe3952de6757e24d0d35a22cbc Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:53:37 -0400 Subject: [PATCH 7/9] Document VideoMotion class - Rename video_sensing/lib.js to video_sensing/library.js --- .../scratch3_video_sensing/debug.js | 2 +- .../scratch3_video_sensing/index.js | 2 +- src/extensions/scratch3_video_sensing/lib.js | 248 ------------ .../scratch3_video_sensing/library.js | 380 ++++++++++++++++++ 4 files changed, 382 insertions(+), 250 deletions(-) delete mode 100644 src/extensions/scratch3_video_sensing/lib.js create mode 100644 src/extensions/scratch3_video_sensing/library.js diff --git a/src/extensions/scratch3_video_sensing/debug.js b/src/extensions/scratch3_video_sensing/debug.js index 24603b584..0dd3917ef 100644 --- a/src/extensions/scratch3_video_sensing/debug.js +++ b/src/extensions/scratch3_video_sensing/debug.js @@ -4,7 +4,7 @@ * @file debug.js */ -const VideoMotion = require('./lib'); +const VideoMotion = require('./library'); const VideoMotionView = require('./view'); module.exports = { diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 10658f0c6..a59daf489 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -5,7 +5,7 @@ const BlockType = require('../../extension-support/block-type'); const Clone = require('../../util/clone'); const log = require('../../util/log'); -const VideoMotion = require('./lib'); +const VideoMotion = require('./library'); /** * Icon svg to be displayed at the left edge of each extension block, encoded as a data URI. diff --git a/src/extensions/scratch3_video_sensing/lib.js b/src/extensions/scratch3_video_sensing/lib.js deleted file mode 100644 index 611315e1d..000000000 --- a/src/extensions/scratch3_video_sensing/lib.js +++ /dev/null @@ -1,248 +0,0 @@ -/** - * lib.js - * - * Tony Hwang and John Maloney, January 2011 - * Michael "Z" Goddard, March 2018 - * - * Video motion sensing primitives. - */ - -const TO_DEGREE = 180 / Math.PI; -const WIDTH = 480; -const HEIGHT = 360; -// chosen empirically to give a range of roughly 0-100 -const AMOUNT_SCALE = 100; -// note 2e-4 * activePixelNum is an experimentally tuned threshold for my -// logitech Pro 9000 webcam - TTH -const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4; -const THRESHOLD = 10; -const WINSIZE = 8; -const LOCAL_MAX_AMOUNT = 100; -const LOCAL_THRESHOLD = THRESHOLD / 3; - -const STATE_KEY = 'Scratch.videoSensing'; - -class VideoMotion { - constructor () { - this.frameNumber = 0; - this.motionAmount = 0; - this.motionDirection = 0; - this.analysisDone = false; - - this.curr = null; - this.prev = null; - - this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1); - this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT); - this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT); - } - - reset () { - this.prev = this.curr = null; - this.motionAmount = this.motionDirection = 0; - this.analysisDone = true; - - const targets = this.runtime.targets; - for (let i = 0; i < targets.length; i++) { - targets[i].getCustomState(STATE_KEY).motionAmount = 0; - targets[i].getCustomState(STATE_KEY).motionDirection = 0; - } - } - - addFrame (source) { - this.frameNumber++; - - this.prev = this.curr; - this.curr = new Uint32Array(source.buffer.slice()); - - const _tmp = this._prev; - this._prev = this._curr; - this._curr = _tmp; - for (let i = 0; i < this.curr.length; i++) { - this._curr[i] = this.curr[i] & 0xff; - } - - this.analysisDone = false; - } - - analyzeFrame () { - if (!this.curr || !this.prev) { - this.motionAmount = this.motionDirection = -1; - // don't have two frames to analyze yet - return; - } - - const { - _curr: curr, - _prev: prev - } = this; - - const winStep = (WINSIZE * 2) + 1; - const wmax = WIDTH - WINSIZE - 1; - const hmax = HEIGHT - WINSIZE - 1; - - let uu = 0; - let vv = 0; - let n = 0; - - for (let i = WINSIZE + 1; i < hmax; i += winStep) { - for (let j = WINSIZE + 1; j < wmax; j += winStep) { - let A2 = 0; - let A1B2 = 0; - let B1 = 0; - let C1 = 0; - let C2 = 0; - - let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE; - let nextAddress = address + winStep; - const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE; - for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) { - for (; address <= nextAddress; address += 1) { - const gradT = ((prev[address]) - (curr[address])); - const gradX = ((curr[address - 1]) - (curr[address + 1])); - const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); - - A2 += gradX * gradX; - A1B2 += gradX * gradY; - B1 += gradY * gradY; - C2 += gradX * gradT; - C1 += gradY * gradT; - } - } - - const delta = ((A1B2 * A1B2) - (A2 * B1)); - let u = 0; - let v = 0; - if (delta) { - // system is not singular - solving by Kramer method - const deltaX = -((C1 * A1B2) - (C2 * B1)); - const deltaY = -((A1B2 * C2) - (A2 * C1)); - const Idelta = 8 / delta; - u = deltaX * Idelta; - v = deltaY * Idelta; - } else { - // singular system - find optical flow in gradient direction - const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); - if (Norm) { - const IGradNorm = 8 / Norm; - const temp = -(C1 + C2) * IGradNorm; - u = (A1B2 + A2) * temp; - v = (B1 + A1B2) * temp; - } - } - - if (-winStep < u && u < winStep && -winStep < v && v < winStep) { - uu += u; - vv += v; - n++; - } - } - } - - uu /= n; - vv /= n; - this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv)); - if (this.motionAmount > THRESHOLD) { - // Scratch direction - this.motionDirection = (((Math.atan2(vv, uu) * TO_DEGREE) + 270) % 360) - 180; - } - this.analysisDone = true; - } - - getLocalMotion (drawable, state) { - if (!this.curr || !this.prev) { - state.motionAmount = state.motionDirection = -1; - // don't have two frames to analyze yet - return; - } - if (state.motionFrameNumber !== this.frameNumber) { - const { - _prev: prev, - _curr: curr - } = this; - - const boundingRect = drawable.getFastBounds(); - const xmin = Math.floor(boundingRect.left + (WIDTH / 2)); - const xmax = Math.floor(boundingRect.right + (WIDTH / 2)); - const ymin = Math.floor((HEIGHT / 2) - boundingRect.top); - const ymax = Math.floor((HEIGHT / 2) - boundingRect.bottom); - - let A2 = 0; - let A1B2 = 0; - let B1 = 0; - let C1 = 0; - let C2 = 0; - let scaleFactor = 0; - - const position = [0, 0, 0]; - - for (let i = ymin; i < ymax; i++) { - for (let j = xmin; j < xmax; j++) { - position[0] = j - (WIDTH / 2); - position[1] = (HEIGHT / 2) - i; - if ( - j > 0 && (j < WIDTH - 1) && - i > 0 && (i < HEIGHT - 1) && - drawable.isTouching(position) - ) { - const address = (i * WIDTH) + j; - const gradT = ((prev[address]) - (curr[address])); - const gradX = ((curr[address - 1]) - (curr[address + 1])); - const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); - - A2 += gradX * gradX; - A1B2 += gradX * gradY; - B1 += gradY * gradY; - C2 += gradX * gradT; - C1 += gradY * gradT; - scaleFactor++; - } - } - } - - const delta = ((A1B2 * A1B2) - (A2 * B1)); - let u = 0; - let v = 0; - if (delta) { - // system is not singular - solving by Kramer method - const deltaX = -((C1 * A1B2) - (C2 * B1)); - const deltaY = -((A1B2 * C2) - (A2 * C1)); - const Idelta = 8 / delta; - u = deltaX * Idelta; - v = deltaY * Idelta; - } else { - // singular system - find optical flow in gradient direction - const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); - if (Norm) { - const IGradNorm = 8 / Norm; - const temp = -(C1 + C2) * IGradNorm; - u = (A1B2 + A2) * temp; - v = (B1 + A1B2) * temp; - } - } - - let activePixelNum = 0; - if (scaleFactor) { - // store the area of the sprite in pixels - activePixelNum = scaleFactor; - scaleFactor /= (2 * WINSIZE * 2 * WINSIZE); - - u = u / scaleFactor; - v = v / scaleFactor; - } - - state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v)); - if (state.motionAmount > LOCAL_MAX_AMOUNT) { - // clip all magnitudes greater than 100 - state.motionAmount = LOCAL_MAX_AMOUNT; - } - if (state.motionAmount > LOCAL_THRESHOLD) { - // Scratch direction - state.motionDirection = (((Math.atan2(v, u) * TO_DEGREE) + 270) % 360) - 180; - } - state.motionFrameNumber = this.frameNumber; - } - } -} - -module.exports = VideoMotion; diff --git a/src/extensions/scratch3_video_sensing/library.js b/src/extensions/scratch3_video_sensing/library.js new file mode 100644 index 000000000..51a7deef5 --- /dev/null +++ b/src/extensions/scratch3_video_sensing/library.js @@ -0,0 +1,380 @@ +/** + * @file library.js + * + * Tony Hwang and John Maloney, January 2011 + * Michael "Z" Goddard, March 2018 + * + * Video motion sensing primitives. + */ + +const {motionVector, scratchAtan2} = require('./math'); + +/** + * The width of the intended resolution to analyze for motion. + * @type {number} + */ +const WIDTH = 480; + +/** + * The height of the intended resolution to analyze for motion. + * @type {number} + */ +const HEIGHT = 360; + +/** + * A constant value to scale the magnitude of the x and y components called u + * and v. This creates the motionAmount value. + * + * Old note: chosen empirically to give a range of roughly 0-100 + * + * @type {number} + */ +const AMOUNT_SCALE = 100; + +/** + * A constant value to scale the magnitude of the x and y components called u + * and v in the local motion derivative. This creates the motionAmount value on + * a target's motion state. + * + * Old note: note 2e-4 * activePixelNum is an experimentally tuned threshold + * for my logitech Pro 9000 webcam - TTH + * + * @type {number} + */ +const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4; + +/** + * The motion amount must be higher than the THRESHOLD to calculate a new + * direction value. + * @type {number} + */ +const THRESHOLD = 10; + +/** + * The size of the radius of the window of summarized values when considering + * the motion inside the full resolution of the sample. + * @type {number} + */ +const WINSIZE = 8; + +/** + * A ceiling for the motionAmount stored to a local target's motion state. The + * motionAmount is not allowed to be larger than LOCAL_MAX_AMOUNT. + * @type {number} + */ +const LOCAL_MAX_AMOUNT = 100; + +/** + * The motion amount for a target's local motion must be higher than the + * LOCAL_THRESHOLD to calculate a new direction value. + * @type {number} + */ +const LOCAL_THRESHOLD = THRESHOLD / 3; + +/** + * Store the necessary image pixel data to compares frames of a video and + * detect an amount and direction of motion in the full sample or in a + * specified area. + * @constructor + */ +class VideoMotion { + constructor () { + /** + * The number of frames that have been added from a source. + * @type {number} + */ + this.frameNumber = 0; + + /** + * The frameNumber last analyzed. + * @type {number} + */ + this.lastAnalyzedFrame = 0; + + /** + * The amount of motion detected in the current frame. + * @type {number} + */ + this.motionAmount = 0; + + /** + * The direction the motion detected in the frame is general moving in. + * @type {number} + */ + this.motionDirection = 0; + + /** + * A copy of the current frame's pixel values. A index of the array is + * represented in RGBA. The lowest byte is red. The next is green. The + * next is blue. And the last is the alpha value of that pixel. + * @type {Uint32Array} + */ + this.curr = null; + + /** + * A copy of the last frame's pixel values. + * @type {Uint32Array} + */ + this.prev = null; + + /** + * A buffer for holding one component of a pixel's full value twice. + * One for the current value. And one for the last value. + * @type {number} + */ + this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1); + + /** + * A clamped uint8 view of _arrays. One component of each index of the + * curr member is copied into this array. + * @type {number} + */ + this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT); + + /** + * A clamped uint8 view of _arrays. One component of each index of the + * prev member is copied into this array. + * @type {number} + */ + this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT); + } + + /** + * Reset internal state so future frame analysis does not consider values + * from before this method was called. + */ + reset () { + this.frameNumber = 0; + this.lastAnalyzedFrame = 0; + this.motionAmount = this.motionDirection = 0; + this.prev = this.curr = null; + } + + /** + * Add a frame to be next analyzed. The passed array represent a pixel with + * each index in the RGBA format. + * @param {Uint32Array} source - a source frame of pixels to copy + */ + addFrame (source) { + this.frameNumber++; + + // Swap curr to prev. + this.prev = this.curr; + // Create a clone of the array so any modifications made to the source + // array do not affect the work done in here. + this.curr = new Uint32Array(source.buffer.slice()); + + // Swap _prev and _curr. Copy one of the color components of the new + // array into _curr overwriting what was the old _prev data. + const _tmp = this._prev; + this._prev = this._curr; + this._curr = _tmp; + for (let i = 0; i < this.curr.length; i++) { + this._curr[i] = this.curr[i] & 0xff; + } + } + + /** + * Analyze the current frame against the previous frame determining the + * amount of motion and direction of the motion. + */ + analyzeFrame () { + if (!this.curr || !this.prev) { + this.motionAmount = this.motionDirection = -1; + // Don't have two frames to analyze yet + return; + } + + // Return early if new data has not been received. + if (this.lastAnalyzedFrame === this.frameNumber) { + return; + } + this.lastAnalyzedFrame = this.frameNumber; + + const { + _curr: curr, + _prev: prev + } = this; + + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + // Accumulate 2d motion vectors from groups of pixels and average it + // later. + let uu = 0; + let vv = 0; + let n = 0; + + // Iterate over groups of cells building up the components to determine + // a motion vector for each cell instead of the whole frame to avoid + // integer overflows. + for (let i = WINSIZE + 1; i < hmax; i += winStep) { + for (let j = WINSIZE + 1; j < wmax; j += winStep) { + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C1 = 0; + let C2 = 0; + + // This is a performance critical math region. + let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE; + let nextAddress = address + winStep; + const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE; + for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) { + for (; address <= nextAddress; address += 1) { + // The difference in color between the last frame and + // the current frame. + const gradT = ((prev[address]) - (curr[address])); + // The difference between the pixel to the left and the + // pixel to the right. + const gradX = ((curr[address - 1]) - (curr[address + 1])); + // The difference between the pixel above and the pixel + // below. + const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); + + // Add the combined values of this pixel to previously + // considered pixels. + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + } + } + + // Use the accumalated values from the for loop to determine a + // motion direction. + const {u, v} = motionVector(A2, A1B2, B1, C2, C1); + + // If u and v are within negative winStep to positive winStep, + // add them to a sum that will later be averaged. + if (-winStep < u && u < winStep && -winStep < v && v < winStep) { + uu += u; + vv += v; + n++; + } + } + } + + // Average the summed vector values of all of the motion groups. + uu /= n; + vv /= n; + + // Scale the magnitude of the averaged UV vector. + this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv)); + if (this.motionAmount > THRESHOLD) { + // Scratch direction + this.motionDirection = scratchAtan2(vv, uu); + } + } + + /** + * Build motion amount and direction values based on stored current and + * previous frame that overlaps a given drawable. + * @param {Drawable} drawable - touchable and bounded drawable to build motion for + * @param {MotionState} state - state to store built values to + */ + getLocalMotion (drawable, state) { + if (!this.curr || !this.prev) { + state.motionAmount = state.motionDirection = -1; + // Don't have two frames to analyze yet + return; + } + + // Skip if the current frame has already been considered for this state. + if (state.motionFrameNumber !== this.frameNumber) { + const { + _prev: prev, + _curr: curr + } = this; + + // Restrict the region the amount and direction are built from to + // the area of the current frame overlapped by the given drawable's + // bounding box. + const boundingRect = drawable.getFastBounds(); + // Transform the bounding box from scratch space to a space from 0, + // 0 to WIDTH, HEIGHT. + const xmin = Math.max(Math.floor(boundingRect.left + (WIDTH / 2)), 1); + const xmax = Math.min(Math.floor(boundingRect.right + (WIDTH / 2)), WIDTH - 1); + const ymin = Math.max(Math.floor((HEIGHT / 2) - boundingRect.top), 1); + const ymax = Math.min(Math.floor((HEIGHT / 2) - boundingRect.bottom), HEIGHT - 1); + + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C1 = 0; + let C2 = 0; + let scaleFactor = 0; + + const position = [0, 0, 0]; + + // This is a performance critical math region. + for (let i = ymin; i < ymax; i++) { + for (let j = xmin; j < xmax; j++) { + // i and j are in a coordinate planning ranging from 0 to + // HEIGHT and 0 to WIDTH. Transform that into Scratch's + // range of HEIGHT / 2 to -HEIGHT / 2 and -WIDTH / 2 to + // WIDTH / 2; + position[0] = j - (WIDTH / 2); + position[1] = (HEIGHT / 2) - i; + // Consider only pixels in the drawable that can touch the + // edge or other drawables. Empty space in the current skin + // is skipped. + if (drawable.isTouching(position)) { + const address = (i * WIDTH) + j; + // The difference in color between the last frame and + // the current frame. + const gradT = ((prev[address]) - (curr[address])); + // The difference between the pixel to the left and the + // pixel to the right. + const gradX = ((curr[address - 1]) - (curr[address + 1])); + // The difference between the pixel above and the pixel + // below. + const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); + + // Add the combined values of this pixel to previously + // considered pixels. + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + scaleFactor++; + } + } + } + + // Use the accumalated values from the for loop to determine a + // motion direction. + let {u, v} = motionVector(A2, A1B2, B1, C2, C1); + + let activePixelNum = 0; + if (scaleFactor) { + // Store the area of the sprite in pixels + activePixelNum = scaleFactor; + + scaleFactor /= (2 * WINSIZE * 2 * WINSIZE); + u = u / scaleFactor; + v = v / scaleFactor; + } + + // Scale the magnitude of the averaged UV vector and the number of + // overlapping drawable pixels. + state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v)); + if (state.motionAmount > LOCAL_MAX_AMOUNT) { + // Clip all magnitudes greater than 100. + state.motionAmount = LOCAL_MAX_AMOUNT; + } + if (state.motionAmount > LOCAL_THRESHOLD) { + // Scratch direction. + state.motionDirection = scratchAtan2(v, u); + } + + // Skip future calls on this state until a new frame is added. + state.motionFrameNumber = this.frameNumber; + } + } +} + +module.exports = VideoMotion; From 6d2c29530fd1c2e6c0fa493f579334654ec6a3be Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:54:28 -0400 Subject: [PATCH 8/9] Document VideoMotionView --- src/extensions/scratch3_video_sensing/view.js | 300 ++++++++++++++---- 1 file changed, 242 insertions(+), 58 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/view.js b/src/extensions/scratch3_video_sensing/view.js index 9a50c8ea4..35623a651 100644 --- a/src/extensions/scratch3_video_sensing/view.js +++ b/src/extensions/scratch3_video_sensing/view.js @@ -1,41 +1,207 @@ +const {motionVector} = require('./math'); + const WIDTH = 480; const HEIGHT = 360; const WINSIZE = 8; const AMOUNT_SCALE = 100; const THRESHOLD = 10; +/** + * Modes of debug output that can be rendered. + * @type {object} + */ const OUTPUT = { + /** + * Render the original input. + * @type {number} + */ INPUT: -1, + + /** + * Render the difference of neighboring pixels for each pixel. The + * horizontal difference, or x value, renders in the red output component. + * The vertical difference, or y value, renders in the green output + * component. Pixels with equal neighbors with a kind of lime green or + * #008080 in a RGB hex value. Colors with more red have a lower value to + * the right than the value to the left. Colors with less red have a higher + * value to the right than the value to the left. Similarly colors with + * more green have lower values below than above and colors with less green + * have higher values below than above. + * @type {number} + */ XY: 0, + + /** + * Render the XY output with groups of pixels averaged together. The group + * shape and size matches the full frame's analysis window size. + * @type {number} + */ XY_CELL: 1, + + /** + * Render three color components matching the detection algorith's values + * that multiple the horizontal difference, or x value, and the vertical + * difference, or y value together. The red component is the x value + * squared. The green component is the y value squared. The blue component + * is the x value times the y value. The detection code refers to these + * values as A2, B1, and A1B2. + * @type {number} + */ AB: 2, + + /** + * Render the AB output of groups of pixels summarized by their combined + * square root. The group shape and size matches the full frame's analysis + * window size. + * @type {number} + */ AB_CELL: 3, + + /** + * Render a single color component matching the temporal difference or the + * difference in color for the same pixel coordinate in the current frame + * and the last frame. The difference is rendered in the blue color + * component since x and y axis differences tend to use red and green. + * @type {number} + */ T: 4, + + /** + * Render the T output of groups of pixels averaged. The group shape and + * size matches the full frame's analysis window. + * @type {number} + */ T_CELL: 5, + + /** + * Render the XY and T outputs together. The x and y axis values use the + * red and green color components as they do in the XY output. The t values + * use the blue color component as the T output does. + * @type {number} + */ XYT: 6, + + /** + * Render the XYT output of groups of pixels averaged. The group shape and + * size matches the full frame's analysis window. + * @type {number} + */ XYT_CELL: 7, + + /** + * Render the horizontal pixel difference times the temporal difference as + * red and the vertical and temporal difference as green. Multiplcation of + * these values ends up with sharp differences in the output showing edge + * details where motion is happening. + * @type {number} + */ C: 8, + + /** + * Render the C output of groups of pixels averaged. The group shape and + * size matches the full frame's analysis window. + * @type {number} + */ C_CELL: 9, - UV_CELL: 10 + + /** + * Render a per pixel version of UV_CELL. UV_CELL is a close to final step + * of the motion code that builds a motion amount and direction from those + * values. UV_CELL renders grouped summarized values, UV does the per pixel + * version but its can only represent one motion vector code path out of + * two choices. Determining the motion vector compares some of the built + * values but building the values with one pixel ensures this first + * comparison says the values are equal. Even though only one code path is + * used to build the values, its output is close to approximating the + * better solution building vectors from groups of pixels to help + * illustrate when the code determines the motion amount and direction to + * be. + * @type {number} + */ + UV: 10, + + /** + * Render cells of mulitple pixels at a step in the motion code that has + * the same cell values and turns them into motion vectors showing the + * amount of motion in the x axis and y axis separately. Those values are a + * step away from becoming a motion amount and direction through standard + * vector to magnitude and angle values. + * @type {number} + */ + UV_CELL: 11 }; +/** + * Temporary storage structure for returning values in + * VideoMotionView._components. + * @type {object} + */ +const _videoMotionViewComponentsTmp = { + A2: 0, + A1B2: 0, + B1: 0, + C2: 0, + C1: 0 +}; + +/** + * Manage a debug canvas with VideoMotion input frames running parts of what + * VideoMotion does to visualize what it does. + * @param {VideoMotion} motion - VideoMotion with inputs to visualize + * @param {OUTPUT} output - visualization output mode + * @constructor + */ class VideoMotionView { constructor (motion, output = OUTPUT.XYT) { + /** + * VideoMotion instance to visualize. + * @type {VideoMotion} + */ this.motion = motion; + /** + * Debug canvas to render to. + * @type {HTMLCanvasElement} + */ const canvas = this.canvas = document.createElement('canvas'); canvas.width = WIDTH; canvas.height = HEIGHT; + + /** + * 2D context to draw to debug canvas. + * @type {CanvasRendering2DContext} + */ this.context = canvas.getContext('2d'); + /** + * Visualization output mode. + * @type {OUTPUT} + */ this.output = output; + + /** + * Pixel buffer to store output values into before they replace the last frames info in the debug canvas. + * @type {Uint32Array} + */ this.buffer = new Uint32Array(WIDTH * HEIGHT); } + /** + * Modes of debug output that can be rendered. + * @type {object} + */ static get OUTPUT () { return OUTPUT; } + /** + * Iterate each pixel address location and call a function with that address. + * @param {number} xStart - start location on the x axis of the output pixel buffer + * @param {number} yStart - start location on the y axis of the output pixel buffer + * @param {nubmer} xStop - location to stop at on the x axis + * @param {number} yStop - location to stop at on the y axis + * @param {function} fn - handle to call with each iterated address + */ _eachAddress (xStart, yStart, xStop, yStop, fn) { for (let i = yStart; i < yStop; i++) { for (let j = xStart; j < xStop; j++) { @@ -45,6 +211,17 @@ class VideoMotionView { } } + /** + * Iterate over cells of pixels and call a function with a function to + * iterate over pixel addresses. + * @param {number} xStart - start location on the x axis + * @param {number} yStart - start lcoation on the y axis + * @param {number} xStop - location to stop at on the x axis + * @param {number} yStop - location to stop at on the y axis + * @param {number} xStep - width of the cells + * @param {number} yStep - height of the cells + * @param {function} fn - function to call with a bound handle to _eachAddress + */ _eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) { const xStep2 = (xStep / 2) | 0; const yStep2 = (yStep / 2) | 0; @@ -61,6 +238,11 @@ class VideoMotionView { } } + /** + * Build horizontal, vertical, and temporal difference of a pixel address. + * @param {number} address - address to build values for + * @returns {object} a object with a gradX, grady, and gradT value + */ _grads (address) { const {curr, prev} = this.motion; const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff); @@ -69,6 +251,41 @@ class VideoMotionView { return {gradX, gradY, gradT}; } + /** + * Build component values used in determining a motion vector for a pixel + * address. + * @param {function} eachAddress - a bound handle to _eachAddress to build + * component values for + * @returns {object} a object with a A2, A1B2, B1, C2, C1 value + */ + _components (eachAddress) { + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C2 = 0; + let C1 = 0; + + eachAddress(address => { + const {gradX, gradY, gradT} = this._grads(address); + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + }); + + _videoMotionViewComponentsTmp.A2 = A2; + _videoMotionViewComponentsTmp.A1B2 = A1B2; + _videoMotionViewComponentsTmp.B1 = B1; + _videoMotionViewComponentsTmp.C2 = C2; + _videoMotionViewComponentsTmp.C1 = C1; + return _videoMotionViewComponentsTmp; + } + + /** + * Visualize the motion code output mode selected for this view to the + * debug canvas. + */ draw () { if (!(this.motion.prev && this.motion.curr)) { return; @@ -198,16 +415,7 @@ class VideoMotionView { const hmax = HEIGHT - WINSIZE - 1; this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { - let C2 = 0; - let C1 = 0; - let n = 0; - - eachAddress(address => { - const {gradX, gradY, gradT} = this._grads(address); - C2 += gradX * gradT; - C1 += gradY * gradT; - n += 1; - }); + let {C2, C1} = this._components(eachAddress); C2 = Math.sqrt(C2); C1 = Math.sqrt(C1); @@ -235,18 +443,7 @@ class VideoMotionView { const hmax = HEIGHT - WINSIZE - 1; this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { - let A2 = 0; - let A1B2 = 0; - let B1 = 0; - let n = 0; - - eachAddress(address => { - const {gradX, gradY} = this._grads(address); - A2 += gradX * gradX; - A1B2 += gradX * gradY; - B1 += gradY * gradY; - n += 1; - }); + let {A2, A1B2, B1} = this._components(eachAddress); A2 = Math.sqrt(A2); A1B2 = Math.sqrt(A1B2); @@ -260,51 +457,38 @@ class VideoMotionView { (A2 & 0xff); }); }); + } else if (this.output === OUTPUT.UV) { + const winStep = (WINSIZE * 2) + 1; + + this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { + const {A2, A1B2, B1, C2, C1} = this._components(fn => fn(address)); + const {u, v} = motionVector(A2, A1B2, B1, C2, C1); + + const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep); + const hypot = Math.hypot(u, v); + const amount = AMOUNT_SCALE * hypot; + + buffer[address] = + (0xff << 24) + + (inRange && amount > THRESHOLD ? + (((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) + + (((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) : + 0x8080 + ); + }); } else if (this.output === OUTPUT.UV_CELL) { const winStep = (WINSIZE * 2) + 1; const wmax = WIDTH - WINSIZE - 1; const hmax = HEIGHT - WINSIZE - 1; this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { - let A2 = 0; - let A1B2 = 0; - let B1 = 0; - let C2 = 0; - let C1 = 0; - - eachAddress(address => { - const {gradX, gradY, gradT} = this._grads(address); - A2 += gradX * gradX; - A1B2 += gradX * gradY; - B1 += gradY * gradY; - C2 += gradX * gradT; - C1 += gradY * gradT; - }); - - const delta = ((A1B2 * A1B2) - (A2 * B1)); - let u = 0; - let v = 0; - if (delta) { - /* system is not singular - solving by Kramer method */ - const deltaX = -((C1 * A1B2) - (C2 * B1)); - const deltaY = -((A1B2 * C2) - (A2 * C1)); - const Idelta = 8 / delta; - u = deltaX * Idelta; - v = deltaY * Idelta; - } else { - /* singular system - find optical flow in gradient direction */ - const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); - if (Norm) { - const IGradNorm = 8 / Norm; - const temp = -(C1 + C2) * IGradNorm; - u = (A1B2 + A2) * temp; - v = (B1 + A1B2) * temp; - } - } + const {A2, A1B2, B1, C2, C1} = this._components(eachAddress); + const {u, v} = motionVector(A2, A1B2, B1, C2, C1); const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep); const hypot = Math.hypot(u, v); const amount = AMOUNT_SCALE * hypot; + eachAddress(address => { buffer[address] = (0xff << 24) + From d4bffcbddb0febf0cdb125173e96bb757cb296d9 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:54:58 -0400 Subject: [PATCH 9/9] Rename playground/motion.html to video-sensing.html --- .../{motion.html => video-sensing.html} | 4 +-- .../{motion.js => video-sensing.js} | 33 ++++++++++++++----- webpack.config.js | 4 +-- 3 files changed, 28 insertions(+), 13 deletions(-) rename src/playground/{motion.html => video-sensing.html} (78%) rename src/playground/{motion.js => video-sensing.js} (77%) diff --git a/src/playground/motion.html b/src/playground/video-sensing.html similarity index 78% rename from src/playground/motion.html rename to src/playground/video-sensing.html index c79deb0e9..d8db0601b 100644 --- a/src/playground/motion.html +++ b/src/playground/video-sensing.html @@ -11,8 +11,8 @@ - + - + diff --git a/src/playground/motion.js b/src/playground/video-sensing.js similarity index 77% rename from src/playground/motion.js rename to src/playground/video-sensing.js index a7db4796a..9493518a5 100644 --- a/src/playground/motion.js +++ b/src/playground/video-sensing.js @@ -1,4 +1,7 @@ (function () { + const BENCHMARK_THROTTLE = 250; + const INTERVAL = 33; + const video = document.createElement('video'); navigator.getUserMedia({ audio: false, @@ -17,18 +20,18 @@ video.height = video.videoHeight; }); }, err => { - /* eslint no-console:0 */ + // eslint-disable-next-line no-console console.log(err); }); - const VideoMotion = window.Scratch3MotionDetect.VideoMotion; - const VideoMotionView = window.Scratch3MotionDetect.VideoMotionView; + const VideoMotion = window.Scratch3VideoSensingDebug.VideoMotion; + const VideoMotionView = window.Scratch3VideoSensingDebug.VideoMotionView; // Create motion detector const motion = new VideoMotion(); // Create debug views that will render different slices of how the detector - // uses the a frame of input. + // uses a frame of input. const OUTPUT = VideoMotionView.OUTPUT; const outputKeys = Object.keys(OUTPUT); const outputValues = Object.values(OUTPUT); @@ -38,6 +41,7 @@ const defaultViews = [OUTPUT.INPUT, OUTPUT.XY_CELL, OUTPUT.T_CELL, OUTPUT.UV_CELL]; + // Add activation toggles for each debug view. const activators = document.createElement('div'); activators.style.userSelect = 'none'; outputValues.forEach((output, index) => { @@ -66,8 +70,14 @@ // Add a text line to display milliseconds per frame, motion value, and // motion direction + const textContainer = document.createElement('div'); + const textHeader = document.createElement('div'); + textHeader.innerText = 'duration (us) :: motion amount :: motion direction'; + textContainer.appendChild(textHeader); const textEl = document.createElement('div'); - document.body.appendChild(textEl); + textEl.innerText = `0 :: 0 :: 0`; + textContainer.appendChild(textEl); + document.body.appendChild(textContainer); let textTimer = Date.now(); // Add the motion debug views to the dom after the text line, so the text @@ -82,7 +92,7 @@ const ctx = tempCanvas.getContext('2d'); const loop = function () { - const timeoutId = setTimeout(loop, 33); + const timeoutId = setTimeout(loop, INTERVAL); try { // Get the bitmap data for the video frame @@ -90,15 +100,20 @@ ctx.drawImage( video, 0, 0, video.width || video.clientWidth, video.height || video.clientHeight, - -480, 0, tempCanvas.width, tempCanvas.height + -tempCanvas.width, 0, tempCanvas.width, tempCanvas.height ); ctx.resetTransform(); const data = ctx.getImageData(0, 0, tempCanvas.width, tempCanvas.height); + // Analyze the latest frame. const b = performance.now(); motion.addFrame(data.data); motion.analyzeFrame(); - if (Date.now() - textTimer > 250) { + + // Every so often update the visible debug numbers with duration in + // microseconds, the amount of motion and the direction of the + // motion. + if (Date.now() - textTimer > BENCHMARK_THROTTLE) { const e = performance.now(); const analyzeDuration = ((e - b) * 1000).toFixed(0); const motionAmount = motion.motionAmount.toFixed(1); @@ -108,7 +123,7 @@ } views.forEach(_view => _view.active && _view.draw()); } catch (error) { - /* eslint no-console:0 */ + // eslint-disable-next-line no-console console.error(error.stack || error); clearTimeout(timeoutId); } diff --git a/webpack.config.js b/webpack.config.js index 9580fc023..ad40a58b4 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -87,7 +87,7 @@ module.exports = [ // Renderer 'scratch-render' ], - 'motion-extension': './src/extensions/scratch3_video_sensing/debug' + 'video-sensing-extension-debug': './src/extensions/scratch3_video_sensing/debug' }, output: { path: path.resolve(__dirname, 'playground'), @@ -101,7 +101,7 @@ module.exports = [ }, { test: require.resolve('./src/extensions/scratch3_video_sensing/debug.js'), - loader: 'expose-loader?Scratch3MotionDetect' + loader: 'expose-loader?Scratch3VideoSensingDebug' }, { test: require.resolve('stats.js/build/stats.min.js'),