Merge pull request #1007 from mzgoddard/motion-detect-2

Motion detect 2

src/extensions/scratch3_video_sensing/debug.js

@@ -1,4 +1,10 @@
-const VideoMotion = require('./lib');
+/**
+ * A debug "index" module exporting VideoMotion and VideoMotionView to debug
+ * VideoMotion directly.
+ * @file debug.js
+ */
+
+const VideoMotion = require('./library');
 const VideoMotionView = require('./view');
 
 module.exports = {

src/extensions/scratch3_video_sensing/index.js

@@ -1,10 +1,11 @@
+const Runtime = require('../../engine/runtime');
+
 const ArgumentType = require('../../extension-support/argument-type');
 const BlockType = require('../../extension-support/block-type');
 const Clone = require('../../util/clone');
 const log = require('../../util/log');
-const Timer = require('../../util/timer');
 
-const VideoMotion = require('./lib');
+const VideoMotion = require('./library');
 
 /**
  * Icon svg to be displayed at the left edge of each extension block, encoded as a data URI.
@@ -33,32 +34,123 @@ class Scratch3VideoSensingBlocks {
          */
         this.runtime = runtime;
 
+        /**
+         * The motion detection algoritm used to power the motion amount and
+         * direction values.
+         * @type {VideoMotion}
+         */
         this.detect = new VideoMotion();
 
+        /**
+         * The last millisecond epoch timestamp that the video stream was
+         * analyzed.
+         * @type {number}
+         */
         this._lastUpdate = null;
 
+        /**
+         * Id representing a Scratch Renderer skin the video is rendered to for
+         * previewing.
+         * @type {number}
+         */
         this._skinId = -1;
+
+        /**
+         * The Scratch Renderer Skin object.
+         * @type {Skin}
+         */
         this._skin = null;
+
+        /**
+         * Id for a drawable using the video's skin that will render as a video
+         * preview.
+         * @type {Drawable}
+         */
         this._drawable = -1;
 
+        /**
+         * Canvas DOM element video is rendered to down or up sample to the
+         * expected resolution.
+         * @type {HTMLCanvasElement}
+         */
+        this._sampleCanvas = null;
+
+        /**
+         * Canvas 2D Context to render to the _sampleCanvas member.
+         * @type {CanvasRenderingContext2D}
+         */
+        this._sampleContext = null;
+
+        // Clear target motion state values when the project starts.
+        this.runtime.on(Runtime.PROJECT_RUN_START, this.reset.bind(this));
+
+        // Boot up the video, canvas to down/up sample the video stream, the
+        // preview skin and drawable, and kick off looping the analysis logic.
         this._setupVideo();
         this._setupSampleCanvas();
         this._setupPreview();
         this._loop();
     }
 
-    static get INTERVAL () {
+    /**
-        return 33;
+     * Dimensions the video stream is analyzed at after its rendered to the
-    }
+     * sample canvas.
-
+     * @type {Array.<number>}
+     */
     static get DIMENSIONS () {
         return [480, 360];
     }
 
+    /**
+     * Order preview drawable is inserted at in the renderer.
+     * @type {number}
+     */
     static get ORDER () {
         return 1;
     }
 
+    /**
+     * The key to load & store a target's motion-related state.
+     * @type {string}
+     */
+    static get STATE_KEY () {
+        return 'Scratch.videoSensing';
+    }
+
+    /**
+     * The default motion-related state, to be used when a target has no existing motion state.
+     * @type {MotionState}
+     */
+    static get DEFAULT_MOTION_STATE () {
+        return {
+            motionFrameNumber: 0,
+            motionAmount: 0,
+            motionDirection: 0
+        };
+    }
+
+    /**
+     * Reset the extension's data motion detection data. This will clear out
+     * for example old frames, so the first analyzed frame will not be compared
+     * against a frame from before reset was called.
+     */
+    reset () {
+        this.detect.reset();
+
+        const targets = this.runtime.targets;
+        for (let i = 0; i < targets.length; i++) {
+            const state = targets[i].getCustomState(Scratch3VideoSensingBlocks.STATE_KEY);
+            if (state) {
+                state.motionAmount = 0;
+                state.motionDirection = 0;
+            }
+        }
+    }
+
+    /**
+     * Setup a video element connected to a user media stream.
+     * @private
+     */
     _setupVideo () {
         this._video = document.createElement('video');
         navigator.getUserMedia({
@@ -81,6 +173,11 @@ class Scratch3VideoSensingBlocks {
         });
     }
 
+    /**
+     * Create a campus to render the user media video to down/up sample to the
+     * needed resolution.
+     * @private
+     */
     _setupSampleCanvas () {
         // Create low-resolution image to sample video for analysis and preview
         const canvas = this._sampleCanvas = document.createElement('canvas');
@@ -89,6 +186,11 @@ class Scratch3VideoSensingBlocks {
         this._sampleContext = canvas.getContext('2d');
     }
 
+    /**
+     * Create a Scratch Renderer Skin and Drawable to preview the user media
+     * video stream.
+     * @private
+     */
     _setupPreview () {
         if (this._skinId !== -1) return;
         if (this._skin !== null) return;
@@ -107,6 +209,11 @@ class Scratch3VideoSensingBlocks {
         });
     }
 
+    /**
+     * Occasionally step a loop to sample the video, stamp it to the preview
+     * skin, and add a TypedArray copy of the canvas's pixel data.
+     * @private
+     */
     _loop () {
         setTimeout(this._loop.bind(this), this.runtime.currentStepTime);
 
@@ -165,9 +272,11 @@ class Scratch3VideoSensingBlocks {
     }
 
     /**
-     * Create data for a menu in scratch-blocks format, consisting of an array of objects with text and
+     * Create data for a menu in scratch-blocks format, consisting of an array
-     * value properties. The text is a translated string, and the value is one-indexed.
+     * of objects with text and value properties. The text is a translated
-     * @param  {object[]} info - An array of info objects each having a name property.
+     * string, and the value is one-indexed.
+     * @param {object[]} info - An array of info objects each having a name
+     *   property.
      * @return {array} - An array of objects with text and value properties.
      * @private
      */
@@ -180,27 +289,10 @@ class Scratch3VideoSensingBlocks {
         });
     }
 
-    /**
-     * The key to load & store a target's motion-related state.
-     * @type {string}
-     */
-    static get STATE_KEY () {
-        return 'Scratch.videoSensing';
-    }
-
-    /**
-     * The default music-related state, to be used when a target has no existing music state.
-     * @type {MusicState}
-     */
-    static get DEFAULT_MOTION_STATE () {
-        return {
-            currentInstrument: 0
-        };
-    }
-
     /**
      * @param {Target} target - collect motion state for this target.
-     * @returns {MotionState} the mutable motion state associated with that target. This will be created if necessary.
+     * @returns {MotionState} the mutable motion state associated with that
+     *   target. This will be created if necessary.
      * @private
      */
     _getMotionState (target) {
@@ -213,7 +305,8 @@ class Scratch3VideoSensingBlocks {
     }
 
     /**
-     * An array of info about each drum.
+     * An array of choices of whether a reporter should return the frame's
+     * motion amount or direction.
      * @type {object[]} an array of objects.
      * @param {string} name - the translatable name to display in the drums menu.
      * @param {string} fileName - the name of the audio file containing the drum sound.
@@ -232,8 +325,10 @@ class Scratch3VideoSensingBlocks {
     /**
      * An array of info about each drum.
      * @type {object[]} an array of objects.
-     * @param {string} name - the translatable name to display in the drums menu.
+     * @param {string} name - the translatable name to display in the drums
-     * @param {string} fileName - the name of the audio file containing the drum sound.
+     *   menu.
+     * @param {string} fileName - the name of the audio file containing the
+     *   drum sound.
      */
     get STAGE_SPRITE_INFO () {
         return [
@@ -272,6 +367,19 @@ class Scratch3VideoSensingBlocks {
                             defaultValue: 1
                         }
                     }
+                },
+                {
+                    // @todo this hat needs to be set itself to restart existing
+                    // threads like Scratch 2's behaviour.
+                    opcode: 'whenMotionGreaterThan',
+                    text: 'when video motion > [REFERENCE]',
+                    blockType: BlockType.HAT,
+                    arguments: {
+                        REFERENCE: {
+                            type: ArgumentType.NUMBER,
+                            defaultValue: 10
+                        }
+                    }
                 }
             ],
             menus: {
@@ -281,14 +389,32 @@ class Scratch3VideoSensingBlocks {
         };
     }
 
+    /**
+     * Analyze a part of the frame that a target overlaps.
+     * @param {Target} target - a target to determine where to analyze
+     * @returns {MotionState} the motion state for the given target
+     */
+    _analyzeLocalMotion (target) {
+        const drawable = this.runtime.renderer._allDrawables[target.drawableID];
+        const state = this._getMotionState(target);
+        this.detect.getLocalMotion(drawable, state);
+        return state;
+    }
+
+    /**
+     * A scratch reporter block handle that analyzes the last two frames and
+     * depending on the arguments, returns the motion or direction for the
+     * whole stage or just the target sprite.
+     * @param {object} args - the block arguments
+     * @param {BlockUtility} util - the block utility
+     * @returns {number} the motion amount or direction of the stage or sprite
+     */
     videoOn (args, util) {
         this.detect.analyzeFrame();
 
         let state = this.detect;
         if (Number(args.STAGE_SPRITE) === 2) {
-            const drawable = this.runtime.renderer._allDrawables[util.target.drawableID];
+            state = this._analyzeLocalMotion(util.target);
-            state = this._getMotionState(util.target);
-            this.detect.getLocalMotion(drawable, state);
         }
 
         if (Number(args.MOTION_DIRECTION) === 1) {
@@ -298,38 +424,18 @@ class Scratch3VideoSensingBlocks {
     }
 
     /**
-     * Check if the stack timer needs initialization.
+     * A scratch hat block edge handle that analyzes the last two frames where
-     * @param {object} util - utility object provided by the runtime.
+     * the target sprite overlaps and if it has more motion than the given
-     * @return {boolean} - true if the stack timer needs to be initialized.
+     * reference value.
-     * @private
+     * @param {object} args - the block arguments
+     * @param {BlockUtility} util - the block utility
+     * @returns {boolean} true if the sprite overlaps more motion than the
+     *   reference
      */
-    _stackTimerNeedsInit (util) {
+    whenMotionGreaterThan (args, util) {
-        return !util.stackFrame.timer;
+        this.detect.analyzeFrame();
-    }
+        const state = this._analyzeLocalMotion(util.target);
-
+        return state.motionAmount > Number(args.REFERENCE);
-    /**
-     * Start the stack timer and the yield the thread if necessary.
-     * @param {object} util - utility object provided by the runtime.
-     * @param {number} duration - a duration in seconds to set the timer for.
-     * @private
-     */
-    _startStackTimer (util, duration) {
-        util.stackFrame.timer = new Timer();
-        util.stackFrame.timer.start();
-        util.stackFrame.duration = duration;
-        util.yield();
-    }
-
-    /**
-     * Check the stack timer, and if its time is not up yet, yield the thread.
-     * @param {object} util - utility object provided by the runtime.
-     * @private
-     */
-    _checkStackTimer (util) {
-        const timeElapsed = util.stackFrame.timer.timeElapsed();
-        if (timeElapsed < util.stackFrame.duration * 1000) {
-            util.yield();
-        }
     }
 }
 

src/extensions/scratch3_video_sensing/lib.js

@@ -1,248 +0,0 @@
-/**
- * lib.js
- *
- * Tony Hwang and John Maloney, January 2011
- * Michael "Z" Goddard, March 2018
- *
- * Video motion sensing primitives.
- */
-
-const TO_DEGREE = 180 / Math.PI;
-const WIDTH = 480;
-const HEIGHT = 360;
-// chosen empirically to give a range of roughly 0-100
-const AMOUNT_SCALE = 100;
-// note 2e-4 * activePixelNum is an experimentally tuned threshold for my
-// logitech Pro 9000 webcam - TTH
-const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4;
-const THRESHOLD = 10;
-const WINSIZE = 8;
-const LOCAL_MAX_AMOUNT = 100;
-const LOCAL_THRESHOLD = THRESHOLD / 3;
-
-const STATE_KEY = 'Scratch.videoSensing';
-
-class VideoMotion {
-    constructor () {
-        this.frameNumber = 0;
-        this.motionAmount = 0;
-        this.motionDirection = 0;
-        this.analysisDone = false;
-
-        this.curr = null;
-        this.prev = null;
-
-        this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1);
-        this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT);
-        this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT);
-    }
-
-    reset () {
-        this.prev = this.curr = null;
-        this.motionAmount = this.motionDirection = 0;
-        this.analysisDone = true;
-
-        const targets = this.runtime.targets;
-        for (let i = 0; i < targets.length; i++) {
-            targets[i].getCustomState(STATE_KEY).motionAmount = 0;
-            targets[i].getCustomState(STATE_KEY).motionDirection = 0;
-        }
-    }
-
-    addFrame (source) {
-        this.frameNumber++;
-
-        this.prev = this.curr;
-        this.curr = new Uint32Array(source.buffer.slice());
-
-        const _tmp = this._prev;
-        this._prev = this._curr;
-        this._curr = _tmp;
-        for (let i = 0; i < this.curr.length; i++) {
-            this._curr[i] = this.curr[i] & 0xff;
-        }
-
-        this.analysisDone = false;
-    }
-
-    analyzeFrame () {
-        if (!this.curr || !this.prev) {
-            this.motionAmount = this.motionDirection = -1;
-            // don't have two frames to analyze yet
-            return;
-        }
-
-        const {
-            _curr: curr,
-            _prev: prev
-        } = this;
-
-        const winStep = (WINSIZE * 2) + 1;
-        const wmax = WIDTH - WINSIZE - 1;
-        const hmax = HEIGHT - WINSIZE - 1;
-
-        let uu = 0;
-        let vv = 0;
-        let n = 0;
-
-        for (let i = WINSIZE + 1; i < hmax; i += winStep) {
-            for (let j = WINSIZE + 1; j < wmax; j += winStep) {
-                let A2 = 0;
-                let A1B2 = 0;
-                let B1 = 0;
-                let C1 = 0;
-                let C2 = 0;
-
-                let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE;
-                let nextAddress = address + winStep;
-                const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE;
-                for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) {
-                    for (; address <= nextAddress; address += 1) {
-                        const gradT = ((prev[address]) - (curr[address]));
-                        const gradX = ((curr[address - 1]) - (curr[address + 1]));
-                        const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));
-
-                        A2 += gradX * gradX;
-                        A1B2 += gradX * gradY;
-                        B1 += gradY * gradY;
-                        C2 += gradX * gradT;
-                        C1 += gradY * gradT;
-                    }
-                }
-
-                const delta = ((A1B2 * A1B2) - (A2 * B1));
-                let u = 0;
-                let v = 0;
-                if (delta) {
-                    // system is not singular - solving by Kramer method
-                    const deltaX = -((C1 * A1B2) - (C2 * B1));
-                    const deltaY = -((A1B2 * C2) - (A2 * C1));
-                    const Idelta = 8 / delta;
-                    u = deltaX * Idelta;
-                    v = deltaY * Idelta;
-                } else {
-                    // singular system - find optical flow in gradient direction
-                    const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
-                    if (Norm) {
-                        const IGradNorm = 8 / Norm;
-                        const temp = -(C1 + C2) * IGradNorm;
-                        u = (A1B2 + A2) * temp;
-                        v = (B1 + A1B2) * temp;
-                    }
-                }
-
-                if (-winStep < u && u < winStep && -winStep < v && v < winStep) {
-                    uu += u;
-                    vv += v;
-                    n++;
-                }
-            }
-        }
-
-        uu /= n;
-        vv /= n;
-        this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv));
-        if (this.motionAmount > THRESHOLD) {
-            // Scratch direction
-            this.motionDirection = (((Math.atan2(vv, uu) * TO_DEGREE) + 270) % 360) - 180;
-        }
-        this.analysisDone = true;
-    }
-
-    getLocalMotion (drawable, state) {
-        if (!this.curr || !this.prev) {
-            state.motionAmount = state.motionDirection = -1;
-            // don't have two frames to analyze yet
-            return;
-        }
-        if (state.motionFrameNumber !== this.frameNumber) {
-            const {
-                _prev: prev,
-                _curr: curr
-            } = this;
-
-            const boundingRect = drawable.getFastBounds();
-            const xmin = Math.floor(boundingRect.left + (WIDTH / 2));
-            const xmax = Math.floor(boundingRect.right + (WIDTH / 2));
-            const ymin = Math.floor((HEIGHT / 2) - boundingRect.top);
-            const ymax = Math.floor((HEIGHT / 2) - boundingRect.bottom);
-
-            let A2 = 0;
-            let A1B2 = 0;
-            let B1 = 0;
-            let C1 = 0;
-            let C2 = 0;
-            let scaleFactor = 0;
-
-            const position = [0, 0, 0];
-
-            for (let i = ymin; i < ymax; i++) {
-                for (let j = xmin; j < xmax; j++) {
-                    position[0] = j - (WIDTH / 2);
-                    position[1] = (HEIGHT / 2) - i;
-                    if (
-                        j > 0 && (j < WIDTH - 1) &&
-                        i > 0 && (i < HEIGHT - 1) &&
-                        drawable.isTouching(position)
-                    ) {
-                        const address = (i * WIDTH) + j;
-                        const gradT = ((prev[address]) - (curr[address]));
-                        const gradX = ((curr[address - 1]) - (curr[address + 1]));
-                        const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));
-
-                        A2 += gradX * gradX;
-                        A1B2 += gradX * gradY;
-                        B1 += gradY * gradY;
-                        C2 += gradX * gradT;
-                        C1 += gradY * gradT;
-                        scaleFactor++;
-                    }
-                }
-            }
-
-            const delta = ((A1B2 * A1B2) - (A2 * B1));
-            let u = 0;
-            let v = 0;
-            if (delta) {
-                // system is not singular - solving by Kramer method
-                const deltaX = -((C1 * A1B2) - (C2 * B1));
-                const deltaY = -((A1B2 * C2) - (A2 * C1));
-                const Idelta = 8 / delta;
-                u = deltaX * Idelta;
-                v = deltaY * Idelta;
-            } else {
-                // singular system - find optical flow in gradient direction
-                const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
-                if (Norm) {
-                    const IGradNorm = 8 / Norm;
-                    const temp = -(C1 + C2) * IGradNorm;
-                    u = (A1B2 + A2) * temp;
-                    v = (B1 + A1B2) * temp;
-                }
-            }
-
-            let activePixelNum = 0;
-            if (scaleFactor) {
-                // store the area of the sprite in pixels
-                activePixelNum = scaleFactor;
-                scaleFactor /= (2 * WINSIZE * 2 * WINSIZE);
-
-                u = u / scaleFactor;
-                v = v / scaleFactor;
-            }
-
-            state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v));
-            if (state.motionAmount > LOCAL_MAX_AMOUNT) {
-                // clip all magnitudes greater than 100
-                state.motionAmount = LOCAL_MAX_AMOUNT;
-            }
-            if (state.motionAmount > LOCAL_THRESHOLD) {
-                // Scratch direction
-                state.motionDirection = (((Math.atan2(v, u) * TO_DEGREE) + 270) % 360) - 180;
-            }
-            state.motionFrameNumber = this.frameNumber;
-        }
-    }
-}
-
-module.exports = VideoMotion;

src/extensions/scratch3_video_sensing/library.js

@@ -0,0 +1,380 @@
+/**
+ * @file library.js
+ *
+ * Tony Hwang and John Maloney, January 2011
+ * Michael "Z" Goddard, March 2018
+ *
+ * Video motion sensing primitives.
+ */
+
+const {motionVector, scratchAtan2} = require('./math');
+
+/**
+ * The width of the intended resolution to analyze for motion.
+ * @type {number}
+ */
+const WIDTH = 480;
+
+/**
+ * The height of the intended resolution to analyze for motion.
+ * @type {number}
+ */
+const HEIGHT = 360;
+
+/**
+ * A constant value to scale the magnitude of the x and y components called u
+ * and v. This creates the motionAmount value.
+ *
+ * Old note: chosen empirically to give a range of roughly 0-100
+ *
+ * @type {number}
+ */
+const AMOUNT_SCALE = 100;
+
+/**
+ * A constant value to scale the magnitude of the x and y components called u
+ * and v in the local motion derivative. This creates the motionAmount value on
+ * a target's motion state.
+ *
+ * Old note: note 2e-4 * activePixelNum is an experimentally tuned threshold
+ * for my logitech Pro 9000 webcam - TTH
+ *
+ * @type {number}
+ */
+const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4;
+
+/**
+ * The motion amount must be higher than the THRESHOLD to calculate a new
+ * direction value.
+ * @type {number}
+ */
+const THRESHOLD = 10;
+
+/**
+ * The size of the radius of the window of summarized values when considering
+ * the motion inside the full resolution of the sample.
+ * @type {number}
+ */
+const WINSIZE = 8;
+
+/**
+ * A ceiling for the motionAmount stored to a local target's motion state. The
+ * motionAmount is not allowed to be larger than LOCAL_MAX_AMOUNT.
+ * @type {number}
+ */
+const LOCAL_MAX_AMOUNT = 100;
+
+/**
+ * The motion amount for a target's local motion must be higher than the
+ * LOCAL_THRESHOLD to calculate a new direction value.
+ * @type {number}
+ */
+const LOCAL_THRESHOLD = THRESHOLD / 3;
+
+/**
+ * Store the necessary image pixel data to compares frames of a video and
+ * detect an amount and direction of motion in the full sample or in a
+ * specified area.
+ * @constructor
+ */
+class VideoMotion {
+    constructor () {
+        /**
+         * The number of frames that have been added from a source.
+         * @type {number}
+         */
+        this.frameNumber = 0;
+
+        /**
+         * The frameNumber last analyzed.
+         * @type {number}
+         */
+        this.lastAnalyzedFrame = 0;
+
+        /**
+         * The amount of motion detected in the current frame.
+         * @type {number}
+         */
+        this.motionAmount = 0;
+
+        /**
+         * The direction the motion detected in the frame is general moving in.
+         * @type {number}
+         */
+        this.motionDirection = 0;
+
+        /**
+         * A copy of the current frame's pixel values. A index of the array is
+         * represented in RGBA. The lowest byte is red. The next is green. The
+         * next is blue. And the last is the alpha value of that pixel.
+         * @type {Uint32Array}
+         */
+        this.curr = null;
+
+        /**
+         * A copy of the last frame's pixel values.
+         * @type {Uint32Array}
+         */
+        this.prev = null;
+
+        /**
+         * A buffer for holding one component of a pixel's full value twice.
+         * One for the current value. And one for the last value.
+         * @type {number}
+         */
+        this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1);
+
+        /**
+         * A clamped uint8 view of _arrays. One component of each index of the
+         * curr member is copied into this array.
+         * @type {number}
+         */
+        this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT);
+
+        /**
+         * A clamped uint8 view of _arrays. One component of each index of the
+         * prev member is copied into this array.
+         * @type {number}
+         */
+        this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT);
+    }
+
+    /**
+     * Reset internal state so future frame analysis does not consider values
+     * from before this method was called.
+     */
+    reset () {
+        this.frameNumber = 0;
+        this.lastAnalyzedFrame = 0;
+        this.motionAmount = this.motionDirection = 0;
+        this.prev = this.curr = null;
+    }
+
+    /**
+     * Add a frame to be next analyzed. The passed array represent a pixel with
+     * each index in the RGBA format.
+     * @param {Uint32Array} source - a source frame of pixels to copy
+     */
+    addFrame (source) {
+        this.frameNumber++;
+
+        // Swap curr to prev.
+        this.prev = this.curr;
+        // Create a clone of the array so any modifications made to the source
+        // array do not affect the work done in here.
+        this.curr = new Uint32Array(source.buffer.slice());
+
+        // Swap _prev and _curr. Copy one of the color components of the new
+        // array into _curr overwriting what was the old _prev data.
+        const _tmp = this._prev;
+        this._prev = this._curr;
+        this._curr = _tmp;
+        for (let i = 0; i < this.curr.length; i++) {
+            this._curr[i] = this.curr[i] & 0xff;
+        }
+    }
+
+    /**
+     * Analyze the current frame against the previous frame determining the
+     * amount of motion and direction of the motion.
+     */
+    analyzeFrame () {
+        if (!this.curr || !this.prev) {
+            this.motionAmount = this.motionDirection = -1;
+            // Don't have two frames to analyze yet
+            return;
+        }
+
+        // Return early if new data has not been received.
+        if (this.lastAnalyzedFrame === this.frameNumber) {
+            return;
+        }
+        this.lastAnalyzedFrame = this.frameNumber;
+
+        const {
+            _curr: curr,
+            _prev: prev
+        } = this;
+
+        const winStep = (WINSIZE * 2) + 1;
+        const wmax = WIDTH - WINSIZE - 1;
+        const hmax = HEIGHT - WINSIZE - 1;
+
+        // Accumulate 2d motion vectors from groups of pixels and average it
+        // later.
+        let uu = 0;
+        let vv = 0;
+        let n = 0;
+
+        // Iterate over groups of cells building up the components to determine
+        // a motion vector for each cell instead of the whole frame to avoid
+        // integer overflows.
+        for (let i = WINSIZE + 1; i < hmax; i += winStep) {
+            for (let j = WINSIZE + 1; j < wmax; j += winStep) {
+                let A2 = 0;
+                let A1B2 = 0;
+                let B1 = 0;
+                let C1 = 0;
+                let C2 = 0;
+
+                // This is a performance critical math region.
+                let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE;
+                let nextAddress = address + winStep;
+                const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE;
+                for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) {
+                    for (; address <= nextAddress; address += 1) {
+                        // The difference in color between the last frame and
+                        // the current frame.
+                        const gradT = ((prev[address]) - (curr[address]));
+                        // The difference between the pixel to the left and the
+                        // pixel to the right.
+                        const gradX = ((curr[address - 1]) - (curr[address + 1]));
+                        // The difference between the pixel above and the pixel
+                        // below.
+                        const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));
+
+                        // Add the combined values of this pixel to previously
+                        // considered pixels.
+                        A2 += gradX * gradX;
+                        A1B2 += gradX * gradY;
+                        B1 += gradY * gradY;
+                        C2 += gradX * gradT;
+                        C1 += gradY * gradT;
+                    }
+                }
+
+                // Use the accumalated values from the for loop to determine a
+                // motion direction.
+                const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
+
+                // If u and v are within negative winStep to positive winStep,
+                // add them to a sum that will later be averaged.
+                if (-winStep < u && u < winStep && -winStep < v && v < winStep) {
+                    uu += u;
+                    vv += v;
+                    n++;
+                }
+            }
+        }
+
+        // Average the summed vector values of all of the motion groups.
+        uu /= n;
+        vv /= n;
+
+        // Scale the magnitude of the averaged UV vector.
+        this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv));
+        if (this.motionAmount > THRESHOLD) {
+            // Scratch direction
+            this.motionDirection = scratchAtan2(vv, uu);
+        }
+    }
+
+    /**
+     * Build motion amount and direction values based on stored current and
+     * previous frame that overlaps a given drawable.
+     * @param {Drawable} drawable - touchable and bounded drawable to build motion for
+     * @param {MotionState} state - state to store built values to
+     */
+    getLocalMotion (drawable, state) {
+        if (!this.curr || !this.prev) {
+            state.motionAmount = state.motionDirection = -1;
+            // Don't have two frames to analyze yet
+            return;
+        }
+
+        // Skip if the current frame has already been considered for this state.
+        if (state.motionFrameNumber !== this.frameNumber) {
+            const {
+                _prev: prev,
+                _curr: curr
+            } = this;
+
+            // Restrict the region the amount and direction are built from to
+            // the area of the current frame overlapped by the given drawable's
+            // bounding box.
+            const boundingRect = drawable.getFastBounds();
+            // Transform the bounding box from scratch space to a space from 0,
+            // 0 to WIDTH, HEIGHT.
+            const xmin = Math.max(Math.floor(boundingRect.left + (WIDTH / 2)), 1);
+            const xmax = Math.min(Math.floor(boundingRect.right + (WIDTH / 2)), WIDTH - 1);
+            const ymin = Math.max(Math.floor((HEIGHT / 2) - boundingRect.top), 1);
+            const ymax = Math.min(Math.floor((HEIGHT / 2) - boundingRect.bottom), HEIGHT - 1);
+
+            let A2 = 0;
+            let A1B2 = 0;
+            let B1 = 0;
+            let C1 = 0;
+            let C2 = 0;
+            let scaleFactor = 0;
+
+            const position = [0, 0, 0];
+
+            // This is a performance critical math region.
+            for (let i = ymin; i < ymax; i++) {
+                for (let j = xmin; j < xmax; j++) {
+                    // i and j are in a coordinate planning ranging from 0 to
+                    // HEIGHT and 0 to WIDTH. Transform that into Scratch's
+                    // range of HEIGHT / 2 to -HEIGHT / 2 and -WIDTH / 2 to
+                    // WIDTH / 2;
+                    position[0] = j - (WIDTH / 2);
+                    position[1] = (HEIGHT / 2) - i;
+                    // Consider only pixels in the drawable that can touch the
+                    // edge or other drawables. Empty space in the current skin
+                    // is skipped.
+                    if (drawable.isTouching(position)) {
+                        const address = (i * WIDTH) + j;
+                        // The difference in color between the last frame and
+                        // the current frame.
+                        const gradT = ((prev[address]) - (curr[address]));
+                        // The difference between the pixel to the left and the
+                        // pixel to the right.
+                        const gradX = ((curr[address - 1]) - (curr[address + 1]));
+                        // The difference between the pixel above and the pixel
+                        // below.
+                        const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));
+
+                        // Add the combined values of this pixel to previously
+                        // considered pixels.
+                        A2 += gradX * gradX;
+                        A1B2 += gradX * gradY;
+                        B1 += gradY * gradY;
+                        C2 += gradX * gradT;
+                        C1 += gradY * gradT;
+                        scaleFactor++;
+                    }
+                }
+            }
+
+            // Use the accumalated values from the for loop to determine a
+            // motion direction.
+            let {u, v} = motionVector(A2, A1B2, B1, C2, C1);
+
+            let activePixelNum = 0;
+            if (scaleFactor) {
+                // Store the area of the sprite in pixels
+                activePixelNum = scaleFactor;
+
+                scaleFactor /= (2 * WINSIZE * 2 * WINSIZE);
+                u = u / scaleFactor;
+                v = v / scaleFactor;
+            }
+
+            // Scale the magnitude of the averaged UV vector and the number of
+            // overlapping drawable pixels.
+            state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v));
+            if (state.motionAmount > LOCAL_MAX_AMOUNT) {
+                // Clip all magnitudes greater than 100.
+                state.motionAmount = LOCAL_MAX_AMOUNT;
+            }
+            if (state.motionAmount > LOCAL_THRESHOLD) {
+                // Scratch direction.
+                state.motionDirection = scratchAtan2(v, u);
+            }
+
+            // Skip future calls on this state until a new frame is added.
+            state.motionFrameNumber = this.frameNumber;
+        }
+    }
+}
+
+module.exports = VideoMotion;

src/extensions/scratch3_video_sensing/math.js

@@ -0,0 +1,76 @@
+/**
+ * A constant value helping to transform a value in radians to degrees.
+ * @type {number}
+ */
+const TO_DEGREE = 180 / Math.PI;
+
+/**
+ * A object reused to save on memory allocation returning u and v vector from
+ * motionVector.
+ * @type {UV}
+ */
+const _motionVectorOut = {u: 0, v: 0};
+
+/**
+ * Determine a motion vector combinations of the color component difference on
+ * the x axis, y axis, and temporal axis.
+ * @param {number} A2 - a sum of x axis squared
+ * @param {number} A1B2 - a sum of x axis times y axis
+ * @param {number} B1 - a sum of y axis squared
+ * @param {number} C2 - a sum of x axis times temporal axis
+ * @param {number} C1 - a sum of y axis times temporal axis
+ * @param {UV} out - optional object to store return UV info in
+ * @returns {UV} a uv vector representing the motion for the given input
+ */
+const motionVector = function (A2, A1B2, B1, C2, C1, out = _motionVectorOut) {
+    // Compare sums of X * Y and sums of X squared and Y squared.
+    const delta = ((A1B2 * A1B2) - (A2 * B1));
+    if (delta) {
+        // System is not singular - solving by Kramer method.
+        const deltaX = -((C1 * A1B2) - (C2 * B1));
+        const deltaY = -((A1B2 * C2) - (A2 * C1));
+        const Idelta = 8 / delta;
+        out.u = deltaX * Idelta;
+        out.v = deltaY * Idelta;
+    } else {
+        // Singular system - find optical flow in gradient direction.
+        const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
+        if (Norm) {
+            const IGradNorm = 8 / Norm;
+            const temp = -(C1 + C2) * IGradNorm;
+            out.u = (A1B2 + A2) * temp;
+            out.v = (B1 + A1B2) * temp;
+        } else {
+            out.u = 0;
+            out.v = 0;
+        }
+    }
+    return out;
+};
+
+/**
+ * Translate an angle in degrees with the range -180 to 180 rotated to
+ * Scratch's reference angle.
+ * @param {number} degrees - angle in range -180 to 180
+ * @returns {number} angle from Scratch's reference angle
+ */
+const scratchDegrees = function (degrees) {
+    return ((degrees + 270) % 360) - 180;
+};
+
+/**
+ * Get the angle of the y and x component of a 2d vector in degrees in
+ * Scratch's coordinate plane.
+ * @param {number} y - the y component of a 2d vector
+ * @param {number} x - the x component of a 2d vector
+ * @returns {number} angle in degrees in Scratch's coordinate plane
+ */
+const scratchAtan2 = function (y, x) {
+    return scratchDegrees(Math.atan2(y, x) * TO_DEGREE);
+};
+
+module.exports = {
+    motionVector,
+    scratchDegrees,
+    scratchAtan2
+};

src/extensions/scratch3_video_sensing/view.js

@@ -1,39 +1,207 @@
+const {motionVector} = require('./math');
+
 const WIDTH = 480;
 const HEIGHT = 360;
 const WINSIZE = 8;
 const AMOUNT_SCALE = 100;
 const THRESHOLD = 10;
 
+/**
+ * Modes of debug output that can be rendered.
+ * @type {object}
+ */
 const OUTPUT = {
+    /**
+     * Render the original input.
+     * @type {number}
+     */
     INPUT: -1,
-    XYT: 0,
+
-    XYT_CELL: 1,
+    /**
-    XY: 2,
+     * Render the difference of neighboring pixels for each pixel. The
-    XY_CELL: 3,
+     * horizontal difference, or x value, renders in the red output component.
+     * The vertical difference, or y value, renders in the green output
+     * component. Pixels with equal neighbors with a kind of lime green or
+     * #008080 in a RGB hex value. Colors with more red have a lower value to
+     * the right than the value to the left. Colors with less red have a higher
+     * value to the right than the value to the left. Similarly colors with
+     * more green have lower values below than above and colors with less green
+     * have higher values below than above.
+     * @type {number}
+     */
+    XY: 0,
+
+    /**
+     * Render the XY output with groups of pixels averaged together. The group
+     * shape and size matches the full frame's analysis window size.
+     * @type {number}
+     */
+    XY_CELL: 1,
+
+    /**
+     * Render three color components matching the detection algorith's values
+     * that multiple the horizontal difference, or x value, and the vertical
+     * difference, or y value together. The red component is the x value
+     * squared. The green component is the y value squared. The blue component
+     * is the x value times the y value. The detection code refers to these
+     * values as A2, B1, and A1B2.
+     * @type {number}
+     */
+    AB: 2,
+
+    /**
+     * Render the AB output of groups of pixels summarized by their combined
+     * square root. The group shape and size matches the full frame's analysis
+     * window size.
+     * @type {number}
+     */
+    AB_CELL: 3,
+
+    /**
+     * Render a single color component matching the temporal difference or the
+     * difference in color for the same pixel coordinate in the current frame
+     * and the last frame. The difference is rendered in the blue color
+     * component since x and y axis differences tend to use red and green.
+     * @type {number}
+     */
     T: 4,
+
+    /**
+     * Render the T output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
     T_CELL: 5,
-    C: 6,
+
-    AB: 7,
+    /**
-    UV: 8
+     * Render the XY and T outputs together. The x and y axis values use the
+     * red and green color components as they do in the XY output. The t values
+     * use the blue color component as the T output does.
+     * @type {number}
+     */
+    XYT: 6,
+
+    /**
+     * Render the XYT output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
+    XYT_CELL: 7,
+
+    /**
+     * Render the horizontal pixel difference times the temporal difference as
+     * red and the vertical and temporal difference as green. Multiplcation of
+     * these values ends up with sharp differences in the output showing edge
+     * details where motion is happening.
+     * @type {number}
+     */
+    C: 8,
+
+    /**
+     * Render the C output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
+    C_CELL: 9,
+
+    /**
+     * Render a per pixel version of UV_CELL. UV_CELL is a close to final step
+     * of the motion code that builds a motion amount and direction from those
+     * values. UV_CELL renders grouped summarized values, UV does the per pixel
+     * version but its can only represent one motion vector code path out of
+     * two choices. Determining the motion vector compares some of the built
+     * values but building the values with one pixel ensures this first
+     * comparison says the values are equal. Even though only one code path is
+     * used to build the values, its output is close to approximating the
+     * better solution building vectors from groups of pixels to help
+     * illustrate when the code determines the motion amount and direction to
+     * be.
+     * @type {number}
+     */
+    UV: 10,
+
+    /**
+     * Render cells of mulitple pixels at a step in the motion code that has
+     * the same cell values and turns them into motion vectors showing the
+     * amount of motion in the x axis and y axis separately. Those values are a
+     * step away from becoming a motion amount and direction through standard
+     * vector to magnitude and angle values.
+     * @type {number}
+     */
+    UV_CELL: 11
 };
 
+/**
+ * Temporary storage structure for returning values in
+ * VideoMotionView._components.
+ * @type {object}
+ */
+const _videoMotionViewComponentsTmp = {
+    A2: 0,
+    A1B2: 0,
+    B1: 0,
+    C2: 0,
+    C1: 0
+};
+
+/**
+ * Manage a debug canvas with VideoMotion input frames running parts of what
+ * VideoMotion does to visualize what it does.
+ * @param {VideoMotion} motion - VideoMotion with inputs to visualize
+ * @param {OUTPUT} output - visualization output mode
+ * @constructor
+ */
 class VideoMotionView {
     constructor (motion, output = OUTPUT.XYT) {
+        /**
+         * VideoMotion instance to visualize.
+         * @type {VideoMotion}
+         */
         this.motion = motion;
 
+        /**
+         * Debug canvas to render to.
+         * @type {HTMLCanvasElement}
+         */
         const canvas = this.canvas = document.createElement('canvas');
         canvas.width = WIDTH;
         canvas.height = HEIGHT;
+
+        /**
+         * 2D context to draw to debug canvas.
+         * @type {CanvasRendering2DContext}
+         */
         this.context = canvas.getContext('2d');
 
+        /**
+         * Visualization output mode.
+         * @type {OUTPUT}
+         */
         this.output = output;
+
+        /**
+         * Pixel buffer to store output values into before they replace the last frames info in the debug canvas.
+         * @type {Uint32Array}
+         */
         this.buffer = new Uint32Array(WIDTH * HEIGHT);
     }
 
+    /**
+     * Modes of debug output that can be rendered.
+     * @type {object}
+     */
     static get OUTPUT () {
         return OUTPUT;
     }
 
+    /**
+     * Iterate each pixel address location and call a function with that address.
+     * @param {number} xStart - start location on the x axis of the output pixel buffer
+     * @param {number} yStart - start location on the y axis of the output pixel buffer
+     * @param {nubmer} xStop - location to stop at on the x axis
+     * @param {number} yStop - location to stop at on the y axis
+     * @param {function} fn - handle to call with each iterated address
+     */
     _eachAddress (xStart, yStart, xStop, yStop, fn) {
         for (let i = yStart; i < yStop; i++) {
             for (let j = xStart; j < xStop; j++) {
@@ -43,6 +211,17 @@ class VideoMotionView {
         }
     }
 
+    /**
+     * Iterate over cells of pixels and call a function with a function to
+     * iterate over pixel addresses.
+     * @param {number} xStart - start location on the x axis
+     * @param {number} yStart - start lcoation on the y axis
+     * @param {number} xStop - location to stop at on the x axis
+     * @param {number} yStop - location to stop at on the y axis
+     * @param {number} xStep - width of the cells
+     * @param {number} yStep - height of the cells
+     * @param {function} fn - function to call with a bound handle to _eachAddress
+     */
     _eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) {
         const xStep2 = (xStep / 2) | 0;
         const yStep2 = (yStep / 2) | 0;
@@ -59,6 +238,11 @@ class VideoMotionView {
         }
     }
 
+    /**
+     * Build horizontal, vertical, and temporal difference of a pixel address.
+     * @param {number} address - address to build values for
+     * @returns {object} a object with a gradX, grady, and gradT value
+     */
     _grads (address) {
         const {curr, prev} = this.motion;
         const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff);
@@ -67,6 +251,41 @@ class VideoMotionView {
         return {gradX, gradY, gradT};
     }
 
+    /**
+     * Build component values used in determining a motion vector for a pixel
+     * address.
+     * @param {function} eachAddress - a bound handle to _eachAddress to build
+     *   component values for
+     * @returns {object} a object with a A2, A1B2, B1, C2, C1 value
+     */
+    _components (eachAddress) {
+        let A2 = 0;
+        let A1B2 = 0;
+        let B1 = 0;
+        let C2 = 0;
+        let C1 = 0;
+
+        eachAddress(address => {
+            const {gradX, gradY, gradT} = this._grads(address);
+            A2 += gradX * gradX;
+            A1B2 += gradX * gradY;
+            B1 += gradY * gradY;
+            C2 += gradX * gradT;
+            C1 += gradY * gradT;
+        });
+
+        _videoMotionViewComponentsTmp.A2 = A2;
+        _videoMotionViewComponentsTmp.A1B2 = A1B2;
+        _videoMotionViewComponentsTmp.B1 = B1;
+        _videoMotionViewComponentsTmp.C2 = C2;
+        _videoMotionViewComponentsTmp.C1 = C1;
+        return _videoMotionViewComponentsTmp;
+    }
+
+    /**
+     * Visualize the motion code output mode selected for this view to the
+     * debug canvas.
+     */
     draw () {
         if (!(this.motion.prev && this.motion.curr)) {
             return;
@@ -186,63 +405,90 @@ class VideoMotionView {
                 const {gradX, gradY, gradT} = this._grads(address);
                 buffer[address] =
                     (0xff << 24) +
-                    ((gradY * gradT) << 8) +
+                    (((Math.sqrt(gradY * gradT) * 0x0f) & 0xff) << 8) +
-                    (gradX * gradT);
+                    ((Math.sqrt(gradX * gradT) * 0x0f) & 0xff);
+            });
+        }
+        if (this.output === OUTPUT.C_CELL) {
+            const winStep = (WINSIZE * 2) + 1;
+            const wmax = WIDTH - WINSIZE - 1;
+            const hmax = HEIGHT - WINSIZE - 1;
+
+            this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
+                let {C2, C1} = this._components(eachAddress);
+
+                C2 = Math.sqrt(C2);
+                C1 = Math.sqrt(C1);
+
+                eachAddress(address => {
+                    buffer[address] =
+                        (0xff << 24) +
+                        ((C1 & 0xff) << 8) +
+                        ((C2 & 0xff) << 0);
+                });
             });
         } else if (this.output === OUTPUT.AB) {
             this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
                 const {gradX, gradY} = this._grads(address);
                 buffer[address] =
                     (0xff << 24) +
-                    ((gradX * gradY) << 16) +
+                    (((gradX * gradY) & 0xff) << 16) +
-                    ((gradY * gradY) << 8) +
+                    (((gradY * gradY) & 0xff) << 8) +
-                    (gradX * gradX);
+                    ((gradX * gradX) & 0xff);
             });
-        } else if (this.output === OUTPUT.UV) {
+        }
+        if (this.output === OUTPUT.AB_CELL) {
             const winStep = (WINSIZE * 2) + 1;
             const wmax = WIDTH - WINSIZE - 1;
             const hmax = HEIGHT - WINSIZE - 1;
 
             this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
-                let A2 = 0;
+                let {A2, A1B2, B1} = this._components(eachAddress);
-                let A1B2 = 0;
+
-                let B1 = 0;
+                A2 = Math.sqrt(A2);
-                let C2 = 0;
+                A1B2 = Math.sqrt(A1B2);
-                let C1 = 0;
+                B1 = Math.sqrt(B1);
 
                 eachAddress(address => {
-                    const {gradX, gradY, gradT} = this._grads(address);
+                    buffer[address] =
-                    A2 += gradX * gradX;
+                        (0xff << 24) +
-                    A1B2 += gradX * gradY;
+                        ((A1B2 & 0xff) << 16) +
-                    B1 += gradY * gradY;
+                        ((B1 & 0xff) << 8) +
-                    C2 += gradX * gradT;
+                        (A2 & 0xff);
-                    C1 += gradY * gradT;
                 });
+            });
+        } else if (this.output === OUTPUT.UV) {
+            const winStep = (WINSIZE * 2) + 1;
 
-                const delta = ((A1B2 * A1B2) - (A2 * B1));
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
-                let u = 0;
+                const {A2, A1B2, B1, C2, C1} = this._components(fn => fn(address));
-                let v = 0;
+                const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
-                if (delta) {
-                    /* system is not singular - solving by Kramer method */
-                    const deltaX = -((C1 * A1B2) - (C2 * B1));
-                    const deltaY = -((A1B2 * C2) - (A2 * C1));
-                    const Idelta = 8 / delta;
-                    u = deltaX * Idelta;
-                    v = deltaY * Idelta;
-                } else {
-                    /* singular system - find optical flow in gradient direction */
-                    const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
-                    if (Norm) {
-                        const IGradNorm = 8 / Norm;
-                        const temp = -(C1 + C2) * IGradNorm;
-                        u = (A1B2 + A2) * temp;
-                        v = (B1 + A1B2) * temp;
-                    }
-                }
 
                 const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
                 const hypot = Math.hypot(u, v);
                 const amount = AMOUNT_SCALE * hypot;
+
+                buffer[address] =
+                    (0xff << 24) +
+                    (inRange && amount > THRESHOLD ?
+                        (((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) +
+                        (((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) :
+                        0x8080
+                    );
+            });
+        } else if (this.output === OUTPUT.UV_CELL) {
+            const winStep = (WINSIZE * 2) + 1;
+            const wmax = WIDTH - WINSIZE - 1;
+            const hmax = HEIGHT - WINSIZE - 1;
+
+            this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
+                const {A2, A1B2, B1, C2, C1} = this._components(eachAddress);
+                const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
+
+                const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
+                const hypot = Math.hypot(u, v);
+                const amount = AMOUNT_SCALE * hypot;
+
                 eachAddress(address => {
                     buffer[address] =
                         (0xff << 24) +

src/playground/video-sensing.html

@@ -11,8 +11,8 @@
     <!-- Stage rendering -->
     <script src="./scratch-render.js"></script>
     <!-- Extension -->
-    <script src="./motion-extension.js"></script>
+    <script src="./video-sensing-extension-debug.js"></script>
     <!-- Motion -->
-    <script src="./motion.js"></script>
+    <script src="./video-sensing.js"></script>
 </body>
 </html>

src/playground/video-sensing.js

@@ -1,4 +1,7 @@
 (function () {
+    const BENCHMARK_THROTTLE = 250;
+    const INTERVAL = 33;
+
     const video = document.createElement('video');
     navigator.getUserMedia({
         audio: false,
@@ -17,18 +20,18 @@
             video.height = video.videoHeight;
         });
     }, err => {
-        /* eslint no-console:0 */
+        // eslint-disable-next-line no-console
         console.log(err);
     });
 
-    const VideoMotion = window.Scratch3MotionDetect.VideoMotion;
+    const VideoMotion = window.Scratch3VideoSensingDebug.VideoMotion;
-    const VideoMotionView = window.Scratch3MotionDetect.VideoMotionView;
+    const VideoMotionView = window.Scratch3VideoSensingDebug.VideoMotionView;
 
     // Create motion detector
     const motion = new VideoMotion();
 
     // Create debug views that will render different slices of how the detector
-    // uses the a frame of input.
+    // uses a frame of input.
     const OUTPUT = VideoMotionView.OUTPUT;
     const outputKeys = Object.keys(OUTPUT);
     const outputValues = Object.values(OUTPUT);
@@ -36,8 +39,9 @@
         .map(output => new VideoMotionView(motion, output));
     const view = views[0];
 
-    const defaultViews = [OUTPUT.INPUT, OUTPUT.XY_CELL, OUTPUT.T_CELL, OUTPUT.UV];
+    const defaultViews = [OUTPUT.INPUT, OUTPUT.XY_CELL, OUTPUT.T_CELL, OUTPUT.UV_CELL];
 
+    // Add activation toggles for each debug view.
     const activators = document.createElement('div');
     activators.style.userSelect = 'none';
     outputValues.forEach((output, index) => {
@@ -66,8 +70,14 @@
 
     // Add a text line to display milliseconds per frame, motion value, and
     // motion direction
+    const textContainer = document.createElement('div');
+    const textHeader = document.createElement('div');
+    textHeader.innerText = 'duration (us) :: motion amount :: motion direction';
+    textContainer.appendChild(textHeader);
     const textEl = document.createElement('div');
-    document.body.appendChild(textEl);
+    textEl.innerText = `0 :: 0 :: 0`;
+    textContainer.appendChild(textEl);
+    document.body.appendChild(textContainer);
     let textTimer = Date.now();
 
     // Add the motion debug views to the dom after the text line, so the text
@@ -82,7 +92,7 @@
     const ctx = tempCanvas.getContext('2d');
 
     const loop = function () {
-        const timeoutId = setTimeout(loop, 33);
+        const timeoutId = setTimeout(loop, INTERVAL);
 
         try {
             // Get the bitmap data for the video frame
@@ -90,15 +100,20 @@
             ctx.drawImage(
                 video,
                 0, 0, video.width || video.clientWidth, video.height || video.clientHeight,
-                -480, 0, tempCanvas.width, tempCanvas.height
+                -tempCanvas.width, 0, tempCanvas.width, tempCanvas.height
             );
             ctx.resetTransform();
             const data = ctx.getImageData(0, 0, tempCanvas.width, tempCanvas.height);
 
+            // Analyze the latest frame.
             const b = performance.now();
             motion.addFrame(data.data);
             motion.analyzeFrame();
-            if (Date.now() - textTimer > 250) {
+
+            // Every so often update the visible debug numbers with duration in
+            // microseconds, the amount of motion and the direction of the
+            // motion.
+            if (Date.now() - textTimer > BENCHMARK_THROTTLE) {
                 const e = performance.now();
                 const analyzeDuration = ((e - b) * 1000).toFixed(0);
                 const motionAmount = motion.motionAmount.toFixed(1);
@@ -108,7 +123,7 @@
             }
             views.forEach(_view => _view.active && _view.draw());
         } catch (error) {
-            /* eslint no-console:0 */
+            // eslint-disable-next-line no-console
             console.error(error.stack || error);
             clearTimeout(timeoutId);
         }

webpack.config.js

@@ -87,7 +87,7 @@ module.exports = [
                 // Renderer
                 'scratch-render'
             ],
-            'motion-extension': './src/extensions/scratch3_video_sensing/debug'
+            'video-sensing-extension-debug': './src/extensions/scratch3_video_sensing/debug'
         },
         output: {
             path: path.resolve(__dirname, 'playground'),
@@ -101,7 +101,7 @@ module.exports = [
                 },
                 {
                     test: require.resolve('./src/extensions/scratch3_video_sensing/debug.js'),
-                    loader: 'expose-loader?Scratch3MotionDetect'
+                    loader: 'expose-loader?Scratch3VideoSensingDebug'
                 },
                 {
                     test: require.resolve('stats.js/build/stats.min.js'),