Document VideoMotionView

src/extensions/scratch3_video_sensing/view.js

@@ -1,41 +1,207 @@
+const {motionVector} = require('./math');
+
 const WIDTH = 480;
 const HEIGHT = 360;
 const WINSIZE = 8;
 const AMOUNT_SCALE = 100;
 const THRESHOLD = 10;
 
+/**
+ * Modes of debug output that can be rendered.
+ * @type {object}
+ */
 const OUTPUT = {
+    /**
+     * Render the original input.
+     * @type {number}
+     */
     INPUT: -1,
+
+    /**
+     * Render the difference of neighboring pixels for each pixel. The
+     * horizontal difference, or x value, renders in the red output component.
+     * The vertical difference, or y value, renders in the green output
+     * component. Pixels with equal neighbors with a kind of lime green or
+     * #008080 in a RGB hex value. Colors with more red have a lower value to
+     * the right than the value to the left. Colors with less red have a higher
+     * value to the right than the value to the left. Similarly colors with
+     * more green have lower values below than above and colors with less green
+     * have higher values below than above.
+     * @type {number}
+     */
     XY: 0,
+
+    /**
+     * Render the XY output with groups of pixels averaged together. The group
+     * shape and size matches the full frame's analysis window size.
+     * @type {number}
+     */
     XY_CELL: 1,
+
+    /**
+     * Render three color components matching the detection algorith's values
+     * that multiple the horizontal difference, or x value, and the vertical
+     * difference, or y value together. The red component is the x value
+     * squared. The green component is the y value squared. The blue component
+     * is the x value times the y value. The detection code refers to these
+     * values as A2, B1, and A1B2.
+     * @type {number}
+     */
     AB: 2,
+
+    /**
+     * Render the AB output of groups of pixels summarized by their combined
+     * square root. The group shape and size matches the full frame's analysis
+     * window size.
+     * @type {number}
+     */
     AB_CELL: 3,
+
+    /**
+     * Render a single color component matching the temporal difference or the
+     * difference in color for the same pixel coordinate in the current frame
+     * and the last frame. The difference is rendered in the blue color
+     * component since x and y axis differences tend to use red and green.
+     * @type {number}
+     */
     T: 4,
+
+    /**
+     * Render the T output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
     T_CELL: 5,
+
+    /**
+     * Render the XY and T outputs together. The x and y axis values use the
+     * red and green color components as they do in the XY output. The t values
+     * use the blue color component as the T output does.
+     * @type {number}
+     */
     XYT: 6,
+
+    /**
+     * Render the XYT output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
     XYT_CELL: 7,
+
+    /**
+     * Render the horizontal pixel difference times the temporal difference as
+     * red and the vertical and temporal difference as green. Multiplcation of
+     * these values ends up with sharp differences in the output showing edge
+     * details where motion is happening.
+     * @type {number}
+     */
     C: 8,
+
+    /**
+     * Render the C output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
     C_CELL: 9,
-    UV_CELL: 10
+
+    /**
+     * Render a per pixel version of UV_CELL. UV_CELL is a close to final step
+     * of the motion code that builds a motion amount and direction from those
+     * values. UV_CELL renders grouped summarized values, UV does the per pixel
+     * version but its can only represent one motion vector code path out of
+     * two choices. Determining the motion vector compares some of the built
+     * values but building the values with one pixel ensures this first
+     * comparison says the values are equal. Even though only one code path is
+     * used to build the values, its output is close to approximating the
+     * better solution building vectors from groups of pixels to help
+     * illustrate when the code determines the motion amount and direction to
+     * be.
+     * @type {number}
+     */
+    UV: 10,
+
+    /**
+     * Render cells of mulitple pixels at a step in the motion code that has
+     * the same cell values and turns them into motion vectors showing the
+     * amount of motion in the x axis and y axis separately. Those values are a
+     * step away from becoming a motion amount and direction through standard
+     * vector to magnitude and angle values.
+     * @type {number}
+     */
+    UV_CELL: 11
 };
 
+/**
+ * Temporary storage structure for returning values in
+ * VideoMotionView._components.
+ * @type {object}
+ */
+const _videoMotionViewComponentsTmp = {
+    A2: 0,
+    A1B2: 0,
+    B1: 0,
+    C2: 0,
+    C1: 0
+};
+
+/**
+ * Manage a debug canvas with VideoMotion input frames running parts of what
+ * VideoMotion does to visualize what it does.
+ * @param {VideoMotion} motion - VideoMotion with inputs to visualize
+ * @param {OUTPUT} output - visualization output mode
+ * @constructor
+ */
 class VideoMotionView {
     constructor (motion, output = OUTPUT.XYT) {
+        /**
+         * VideoMotion instance to visualize.
+         * @type {VideoMotion}
+         */
         this.motion = motion;
 
+        /**
+         * Debug canvas to render to.
+         * @type {HTMLCanvasElement}
+         */
         const canvas = this.canvas = document.createElement('canvas');
         canvas.width = WIDTH;
         canvas.height = HEIGHT;
+
+        /**
+         * 2D context to draw to debug canvas.
+         * @type {CanvasRendering2DContext}
+         */
         this.context = canvas.getContext('2d');
 
+        /**
+         * Visualization output mode.
+         * @type {OUTPUT}
+         */
         this.output = output;
+
+        /**
+         * Pixel buffer to store output values into before they replace the last frames info in the debug canvas.
+         * @type {Uint32Array}
+         */
         this.buffer = new Uint32Array(WIDTH * HEIGHT);
     }
 
+    /**
+     * Modes of debug output that can be rendered.
+     * @type {object}
+     */
     static get OUTPUT () {
         return OUTPUT;
     }
 
+    /**
+     * Iterate each pixel address location and call a function with that address.
+     * @param {number} xStart - start location on the x axis of the output pixel buffer
+     * @param {number} yStart - start location on the y axis of the output pixel buffer
+     * @param {nubmer} xStop - location to stop at on the x axis
+     * @param {number} yStop - location to stop at on the y axis
+     * @param {function} fn - handle to call with each iterated address
+     */
     _eachAddress (xStart, yStart, xStop, yStop, fn) {
         for (let i = yStart; i < yStop; i++) {
             for (let j = xStart; j < xStop; j++) {
@@ -45,6 +211,17 @@ class VideoMotionView {
         }
     }
 
+    /**
+     * Iterate over cells of pixels and call a function with a function to
+     * iterate over pixel addresses.
+     * @param {number} xStart - start location on the x axis
+     * @param {number} yStart - start lcoation on the y axis
+     * @param {number} xStop - location to stop at on the x axis
+     * @param {number} yStop - location to stop at on the y axis
+     * @param {number} xStep - width of the cells
+     * @param {number} yStep - height of the cells
+     * @param {function} fn - function to call with a bound handle to _eachAddress
+     */
     _eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) {
         const xStep2 = (xStep / 2) | 0;
         const yStep2 = (yStep / 2) | 0;
@@ -61,6 +238,11 @@ class VideoMotionView {
         }
     }
 
+    /**
+     * Build horizontal, vertical, and temporal difference of a pixel address.
+     * @param {number} address - address to build values for
+     * @returns {object} a object with a gradX, grady, and gradT value
+     */
     _grads (address) {
         const {curr, prev} = this.motion;
         const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff);
@@ -69,6 +251,41 @@ class VideoMotionView {
         return {gradX, gradY, gradT};
     }
 
+    /**
+     * Build component values used in determining a motion vector for a pixel
+     * address.
+     * @param {function} eachAddress - a bound handle to _eachAddress to build
+     *   component values for
+     * @returns {object} a object with a A2, A1B2, B1, C2, C1 value
+     */
+    _components (eachAddress) {
+        let A2 = 0;
+        let A1B2 = 0;
+        let B1 = 0;
+        let C2 = 0;
+        let C1 = 0;
+
+        eachAddress(address => {
+            const {gradX, gradY, gradT} = this._grads(address);
+            A2 += gradX * gradX;
+            A1B2 += gradX * gradY;
+            B1 += gradY * gradY;
+            C2 += gradX * gradT;
+            C1 += gradY * gradT;
+        });
+
+        _videoMotionViewComponentsTmp.A2 = A2;
+        _videoMotionViewComponentsTmp.A1B2 = A1B2;
+        _videoMotionViewComponentsTmp.B1 = B1;
+        _videoMotionViewComponentsTmp.C2 = C2;
+        _videoMotionViewComponentsTmp.C1 = C1;
+        return _videoMotionViewComponentsTmp;
+    }
+
+    /**
+     * Visualize the motion code output mode selected for this view to the
+     * debug canvas.
+     */
     draw () {
         if (!(this.motion.prev && this.motion.curr)) {
             return;
@@ -198,16 +415,7 @@ class VideoMotionView {
             const hmax = HEIGHT - WINSIZE - 1;
 
             this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
-                let C2 = 0;
+                let {C2, C1} = this._components(eachAddress);
-                let C1 = 0;
-                let n = 0;
-
-                eachAddress(address => {
-                    const {gradX, gradY, gradT} = this._grads(address);
-                    C2 += gradX * gradT;
-                    C1 += gradY * gradT;
-                    n += 1;
-                });
 
                 C2 = Math.sqrt(C2);
                 C1 = Math.sqrt(C1);
@@ -235,18 +443,7 @@ class VideoMotionView {
             const hmax = HEIGHT - WINSIZE - 1;
 
             this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
-                let A2 = 0;
+                let {A2, A1B2, B1} = this._components(eachAddress);
-                let A1B2 = 0;
-                let B1 = 0;
-                let n = 0;
-
-                eachAddress(address => {
-                    const {gradX, gradY} = this._grads(address);
-                    A2 += gradX * gradX;
-                    A1B2 += gradX * gradY;
-                    B1 += gradY * gradY;
-                    n += 1;
-                });
 
                 A2 = Math.sqrt(A2);
                 A1B2 = Math.sqrt(A1B2);
@@ -260,51 +457,38 @@ class VideoMotionView {
                         (A2 & 0xff);
                 });
             });
+        } else if (this.output === OUTPUT.UV) {
+            const winStep = (WINSIZE * 2) + 1;
+
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
+                const {A2, A1B2, B1, C2, C1} = this._components(fn => fn(address));
+                const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
+
+                const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
+                const hypot = Math.hypot(u, v);
+                const amount = AMOUNT_SCALE * hypot;
+
+                buffer[address] =
+                    (0xff << 24) +
+                    (inRange && amount > THRESHOLD ?
+                        (((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) +
+                        (((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) :
+                        0x8080
+                    );
+            });
         } else if (this.output === OUTPUT.UV_CELL) {
             const winStep = (WINSIZE * 2) + 1;
             const wmax = WIDTH - WINSIZE - 1;
             const hmax = HEIGHT - WINSIZE - 1;
 
             this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
-                let A2 = 0;
+                const {A2, A1B2, B1, C2, C1} = this._components(eachAddress);
-                let A1B2 = 0;
+                const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
-                let B1 = 0;
-                let C2 = 0;
-                let C1 = 0;
-
-                eachAddress(address => {
-                    const {gradX, gradY, gradT} = this._grads(address);
-                    A2 += gradX * gradX;
-                    A1B2 += gradX * gradY;
-                    B1 += gradY * gradY;
-                    C2 += gradX * gradT;
-                    C1 += gradY * gradT;
-                });
-
-                const delta = ((A1B2 * A1B2) - (A2 * B1));
-                let u = 0;
-                let v = 0;
-                if (delta) {
-                    /* system is not singular - solving by Kramer method */
-                    const deltaX = -((C1 * A1B2) - (C2 * B1));
-                    const deltaY = -((A1B2 * C2) - (A2 * C1));
-                    const Idelta = 8 / delta;
-                    u = deltaX * Idelta;
-                    v = deltaY * Idelta;
-                } else {
-                    /* singular system - find optical flow in gradient direction */
-                    const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
-                    if (Norm) {
-                        const IGradNorm = 8 / Norm;
-                        const temp = -(C1 + C2) * IGradNorm;
-                        u = (A1B2 + A2) * temp;
-                        v = (B1 + A1B2) * temp;
-                    }
-                }
 
                 const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
                 const hypot = Math.hypot(u, v);
                 const amount = AMOUNT_SCALE * hypot;
+
                 eachAddress(address => {
                     buffer[address] =
                         (0xff << 24) +