From 6d2c29530fd1c2e6c0fa493f579334654ec6a3be Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:54:28 -0400 Subject: [PATCH] Document VideoMotionView --- src/extensions/scratch3_video_sensing/view.js | 300 ++++++++++++++---- 1 file changed, 242 insertions(+), 58 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/view.js b/src/extensions/scratch3_video_sensing/view.js index 9a50c8ea4..35623a651 100644 --- a/src/extensions/scratch3_video_sensing/view.js +++ b/src/extensions/scratch3_video_sensing/view.js @@ -1,41 +1,207 @@ +const {motionVector} = require('./math'); + const WIDTH = 480; const HEIGHT = 360; const WINSIZE = 8; const AMOUNT_SCALE = 100; const THRESHOLD = 10; +/** + * Modes of debug output that can be rendered. + * @type {object} + */ const OUTPUT = { + /** + * Render the original input. + * @type {number} + */ INPUT: -1, + + /** + * Render the difference of neighboring pixels for each pixel. The + * horizontal difference, or x value, renders in the red output component. + * The vertical difference, or y value, renders in the green output + * component. Pixels with equal neighbors with a kind of lime green or + * #008080 in a RGB hex value. Colors with more red have a lower value to + * the right than the value to the left. Colors with less red have a higher + * value to the right than the value to the left. Similarly colors with + * more green have lower values below than above and colors with less green + * have higher values below than above. + * @type {number} + */ XY: 0, + + /** + * Render the XY output with groups of pixels averaged together. The group + * shape and size matches the full frame's analysis window size. + * @type {number} + */ XY_CELL: 1, + + /** + * Render three color components matching the detection algorith's values + * that multiple the horizontal difference, or x value, and the vertical + * difference, or y value together. The red component is the x value + * squared. The green component is the y value squared. The blue component + * is the x value times the y value. The detection code refers to these + * values as A2, B1, and A1B2. + * @type {number} + */ AB: 2, + + /** + * Render the AB output of groups of pixels summarized by their combined + * square root. The group shape and size matches the full frame's analysis + * window size. + * @type {number} + */ AB_CELL: 3, + + /** + * Render a single color component matching the temporal difference or the + * difference in color for the same pixel coordinate in the current frame + * and the last frame. The difference is rendered in the blue color + * component since x and y axis differences tend to use red and green. + * @type {number} + */ T: 4, + + /** + * Render the T output of groups of pixels averaged. The group shape and + * size matches the full frame's analysis window. + * @type {number} + */ T_CELL: 5, + + /** + * Render the XY and T outputs together. The x and y axis values use the + * red and green color components as they do in the XY output. The t values + * use the blue color component as the T output does. + * @type {number} + */ XYT: 6, + + /** + * Render the XYT output of groups of pixels averaged. The group shape and + * size matches the full frame's analysis window. + * @type {number} + */ XYT_CELL: 7, + + /** + * Render the horizontal pixel difference times the temporal difference as + * red and the vertical and temporal difference as green. Multiplcation of + * these values ends up with sharp differences in the output showing edge + * details where motion is happening. + * @type {number} + */ C: 8, + + /** + * Render the C output of groups of pixels averaged. The group shape and + * size matches the full frame's analysis window. + * @type {number} + */ C_CELL: 9, - UV_CELL: 10 + + /** + * Render a per pixel version of UV_CELL. UV_CELL is a close to final step + * of the motion code that builds a motion amount and direction from those + * values. UV_CELL renders grouped summarized values, UV does the per pixel + * version but its can only represent one motion vector code path out of + * two choices. Determining the motion vector compares some of the built + * values but building the values with one pixel ensures this first + * comparison says the values are equal. Even though only one code path is + * used to build the values, its output is close to approximating the + * better solution building vectors from groups of pixels to help + * illustrate when the code determines the motion amount and direction to + * be. + * @type {number} + */ + UV: 10, + + /** + * Render cells of mulitple pixels at a step in the motion code that has + * the same cell values and turns them into motion vectors showing the + * amount of motion in the x axis and y axis separately. Those values are a + * step away from becoming a motion amount and direction through standard + * vector to magnitude and angle values. + * @type {number} + */ + UV_CELL: 11 }; +/** + * Temporary storage structure for returning values in + * VideoMotionView._components. + * @type {object} + */ +const _videoMotionViewComponentsTmp = { + A2: 0, + A1B2: 0, + B1: 0, + C2: 0, + C1: 0 +}; + +/** + * Manage a debug canvas with VideoMotion input frames running parts of what + * VideoMotion does to visualize what it does. + * @param {VideoMotion} motion - VideoMotion with inputs to visualize + * @param {OUTPUT} output - visualization output mode + * @constructor + */ class VideoMotionView { constructor (motion, output = OUTPUT.XYT) { + /** + * VideoMotion instance to visualize. + * @type {VideoMotion} + */ this.motion = motion; + /** + * Debug canvas to render to. + * @type {HTMLCanvasElement} + */ const canvas = this.canvas = document.createElement('canvas'); canvas.width = WIDTH; canvas.height = HEIGHT; + + /** + * 2D context to draw to debug canvas. + * @type {CanvasRendering2DContext} + */ this.context = canvas.getContext('2d'); + /** + * Visualization output mode. + * @type {OUTPUT} + */ this.output = output; + + /** + * Pixel buffer to store output values into before they replace the last frames info in the debug canvas. + * @type {Uint32Array} + */ this.buffer = new Uint32Array(WIDTH * HEIGHT); } + /** + * Modes of debug output that can be rendered. + * @type {object} + */ static get OUTPUT () { return OUTPUT; } + /** + * Iterate each pixel address location and call a function with that address. + * @param {number} xStart - start location on the x axis of the output pixel buffer + * @param {number} yStart - start location on the y axis of the output pixel buffer + * @param {nubmer} xStop - location to stop at on the x axis + * @param {number} yStop - location to stop at on the y axis + * @param {function} fn - handle to call with each iterated address + */ _eachAddress (xStart, yStart, xStop, yStop, fn) { for (let i = yStart; i < yStop; i++) { for (let j = xStart; j < xStop; j++) { @@ -45,6 +211,17 @@ class VideoMotionView { } } + /** + * Iterate over cells of pixels and call a function with a function to + * iterate over pixel addresses. + * @param {number} xStart - start location on the x axis + * @param {number} yStart - start lcoation on the y axis + * @param {number} xStop - location to stop at on the x axis + * @param {number} yStop - location to stop at on the y axis + * @param {number} xStep - width of the cells + * @param {number} yStep - height of the cells + * @param {function} fn - function to call with a bound handle to _eachAddress + */ _eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) { const xStep2 = (xStep / 2) | 0; const yStep2 = (yStep / 2) | 0; @@ -61,6 +238,11 @@ class VideoMotionView { } } + /** + * Build horizontal, vertical, and temporal difference of a pixel address. + * @param {number} address - address to build values for + * @returns {object} a object with a gradX, grady, and gradT value + */ _grads (address) { const {curr, prev} = this.motion; const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff); @@ -69,6 +251,41 @@ class VideoMotionView { return {gradX, gradY, gradT}; } + /** + * Build component values used in determining a motion vector for a pixel + * address. + * @param {function} eachAddress - a bound handle to _eachAddress to build + * component values for + * @returns {object} a object with a A2, A1B2, B1, C2, C1 value + */ + _components (eachAddress) { + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C2 = 0; + let C1 = 0; + + eachAddress(address => { + const {gradX, gradY, gradT} = this._grads(address); + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + }); + + _videoMotionViewComponentsTmp.A2 = A2; + _videoMotionViewComponentsTmp.A1B2 = A1B2; + _videoMotionViewComponentsTmp.B1 = B1; + _videoMotionViewComponentsTmp.C2 = C2; + _videoMotionViewComponentsTmp.C1 = C1; + return _videoMotionViewComponentsTmp; + } + + /** + * Visualize the motion code output mode selected for this view to the + * debug canvas. + */ draw () { if (!(this.motion.prev && this.motion.curr)) { return; @@ -198,16 +415,7 @@ class VideoMotionView { const hmax = HEIGHT - WINSIZE - 1; this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { - let C2 = 0; - let C1 = 0; - let n = 0; - - eachAddress(address => { - const {gradX, gradY, gradT} = this._grads(address); - C2 += gradX * gradT; - C1 += gradY * gradT; - n += 1; - }); + let {C2, C1} = this._components(eachAddress); C2 = Math.sqrt(C2); C1 = Math.sqrt(C1); @@ -235,18 +443,7 @@ class VideoMotionView { const hmax = HEIGHT - WINSIZE - 1; this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { - let A2 = 0; - let A1B2 = 0; - let B1 = 0; - let n = 0; - - eachAddress(address => { - const {gradX, gradY} = this._grads(address); - A2 += gradX * gradX; - A1B2 += gradX * gradY; - B1 += gradY * gradY; - n += 1; - }); + let {A2, A1B2, B1} = this._components(eachAddress); A2 = Math.sqrt(A2); A1B2 = Math.sqrt(A1B2); @@ -260,51 +457,38 @@ class VideoMotionView { (A2 & 0xff); }); }); + } else if (this.output === OUTPUT.UV) { + const winStep = (WINSIZE * 2) + 1; + + this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { + const {A2, A1B2, B1, C2, C1} = this._components(fn => fn(address)); + const {u, v} = motionVector(A2, A1B2, B1, C2, C1); + + const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep); + const hypot = Math.hypot(u, v); + const amount = AMOUNT_SCALE * hypot; + + buffer[address] = + (0xff << 24) + + (inRange && amount > THRESHOLD ? + (((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) + + (((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) : + 0x8080 + ); + }); } else if (this.output === OUTPUT.UV_CELL) { const winStep = (WINSIZE * 2) + 1; const wmax = WIDTH - WINSIZE - 1; const hmax = HEIGHT - WINSIZE - 1; this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { - let A2 = 0; - let A1B2 = 0; - let B1 = 0; - let C2 = 0; - let C1 = 0; - - eachAddress(address => { - const {gradX, gradY, gradT} = this._grads(address); - A2 += gradX * gradX; - A1B2 += gradX * gradY; - B1 += gradY * gradY; - C2 += gradX * gradT; - C1 += gradY * gradT; - }); - - const delta = ((A1B2 * A1B2) - (A2 * B1)); - let u = 0; - let v = 0; - if (delta) { - /* system is not singular - solving by Kramer method */ - const deltaX = -((C1 * A1B2) - (C2 * B1)); - const deltaY = -((A1B2 * C2) - (A2 * C1)); - const Idelta = 8 / delta; - u = deltaX * Idelta; - v = deltaY * Idelta; - } else { - /* singular system - find optical flow in gradient direction */ - const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); - if (Norm) { - const IGradNorm = 8 / Norm; - const temp = -(C1 + C2) * IGradNorm; - u = (A1B2 + A2) * temp; - v = (B1 + A1B2) * temp; - } - } + const {A2, A1B2, B1, C2, C1} = this._components(eachAddress); + const {u, v} = motionVector(A2, A1B2, B1, C2, C1); const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep); const hypot = Math.hypot(u, v); const amount = AMOUNT_SCALE * hypot; + eachAddress(address => { buffer[address] = (0xff << 24) +