mirror of
https://github.com/scratchfoundation/scratch-vm.git
synced 2024-12-24 06:52:40 -05:00
Document VideoMotionView
This commit is contained in:
Michael "Z" Goddard
parent
9a2e937271
commit
6d2c29530f
1 changed files with 242 additions and 58 deletions
|
|
@ -1,41 +1,207 @@
|
|||
const {motionVector} = require('./math');
|
||||
|
||||
const WIDTH = 480;
|
||||
const HEIGHT = 360;
|
||||
const WINSIZE = 8;
|
||||
const AMOUNT_SCALE = 100;
|
||||
const THRESHOLD = 10;
|
||||
|
||||
/**
|
||||
* Modes of debug output that can be rendered.
|
||||
* @type {object}
|
||||
*/
|
||||
const OUTPUT = {
|
||||
/**
|
||||
* Render the original input.
|
||||
* @type {number}
|
||||
*/
|
||||
INPUT: -1,
|
||||
|
||||
/**
|
||||
* Render the difference of neighboring pixels for each pixel. The
|
||||
* horizontal difference, or x value, renders in the red output component.
|
||||
* The vertical difference, or y value, renders in the green output
|
||||
* component. Pixels with equal neighbors with a kind of lime green or
|
||||
* #008080 in a RGB hex value. Colors with more red have a lower value to
|
||||
* the right than the value to the left. Colors with less red have a higher
|
||||
* value to the right than the value to the left. Similarly colors with
|
||||
* more green have lower values below than above and colors with less green
|
||||
* have higher values below than above.
|
||||
* @type {number}
|
||||
*/
|
||||
XY: 0,
|
||||
|
||||
/**
|
||||
* Render the XY output with groups of pixels averaged together. The group
|
||||
* shape and size matches the full frame's analysis window size.
|
||||
* @type {number}
|
||||
*/
|
||||
XY_CELL: 1,
|
||||
|
||||
/**
|
||||
* Render three color components matching the detection algorith's values
|
||||
* that multiple the horizontal difference, or x value, and the vertical
|
||||
* difference, or y value together. The red component is the x value
|
||||
* squared. The green component is the y value squared. The blue component
|
||||
* is the x value times the y value. The detection code refers to these
|
||||
* values as A2, B1, and A1B2.
|
||||
* @type {number}
|
||||
*/
|
||||
AB: 2,
|
||||
|
||||
/**
|
||||
* Render the AB output of groups of pixels summarized by their combined
|
||||
* square root. The group shape and size matches the full frame's analysis
|
||||
* window size.
|
||||
* @type {number}
|
||||
*/
|
||||
AB_CELL: 3,
|
||||
|
||||
/**
|
||||
* Render a single color component matching the temporal difference or the
|
||||
* difference in color for the same pixel coordinate in the current frame
|
||||
* and the last frame. The difference is rendered in the blue color
|
||||
* component since x and y axis differences tend to use red and green.
|
||||
* @type {number}
|
||||
*/
|
||||
T: 4,
|
||||
|
||||
/**
|
||||
* Render the T output of groups of pixels averaged. The group shape and
|
||||
* size matches the full frame's analysis window.
|
||||
* @type {number}
|
||||
*/
|
||||
T_CELL: 5,
|
||||
|
||||
/**
|
||||
* Render the XY and T outputs together. The x and y axis values use the
|
||||
* red and green color components as they do in the XY output. The t values
|
||||
* use the blue color component as the T output does.
|
||||
* @type {number}
|
||||
*/
|
||||
XYT: 6,
|
||||
|
||||
/**
|
||||
* Render the XYT output of groups of pixels averaged. The group shape and
|
||||
* size matches the full frame's analysis window.
|
||||
* @type {number}
|
||||
*/
|
||||
XYT_CELL: 7,
|
||||
|
||||
/**
|
||||
* Render the horizontal pixel difference times the temporal difference as
|
||||
* red and the vertical and temporal difference as green. Multiplcation of
|
||||
* these values ends up with sharp differences in the output showing edge
|
||||
* details where motion is happening.
|
||||
* @type {number}
|
||||
*/
|
||||
C: 8,
|
||||
|
||||
/**
|
||||
* Render the C output of groups of pixels averaged. The group shape and
|
||||
* size matches the full frame's analysis window.
|
||||
* @type {number}
|
||||
*/
|
||||
C_CELL: 9,
|
||||
UV_CELL: 10
|
||||
|
||||
/**
|
||||
* Render a per pixel version of UV_CELL. UV_CELL is a close to final step
|
||||
* of the motion code that builds a motion amount and direction from those
|
||||
* values. UV_CELL renders grouped summarized values, UV does the per pixel
|
||||
* version but its can only represent one motion vector code path out of
|
||||
* two choices. Determining the motion vector compares some of the built
|
||||
* values but building the values with one pixel ensures this first
|
||||
* comparison says the values are equal. Even though only one code path is
|
||||
* used to build the values, its output is close to approximating the
|
||||
* better solution building vectors from groups of pixels to help
|
||||
* illustrate when the code determines the motion amount and direction to
|
||||
* be.
|
||||
* @type {number}
|
||||
*/
|
||||
UV: 10,
|
||||
|
||||
/**
|
||||
* Render cells of mulitple pixels at a step in the motion code that has
|
||||
* the same cell values and turns them into motion vectors showing the
|
||||
* amount of motion in the x axis and y axis separately. Those values are a
|
||||
* step away from becoming a motion amount and direction through standard
|
||||
* vector to magnitude and angle values.
|
||||
* @type {number}
|
||||
*/
|
||||
UV_CELL: 11
|
||||
};
|
||||
|
||||
/**
|
||||
* Temporary storage structure for returning values in
|
||||
* VideoMotionView._components.
|
||||
* @type {object}
|
||||
*/
|
||||
const _videoMotionViewComponentsTmp = {
|
||||
A2: 0,
|
||||
A1B2: 0,
|
||||
B1: 0,
|
||||
C2: 0,
|
||||
C1: 0
|
||||
};
|
||||
|
||||
/**
|
||||
* Manage a debug canvas with VideoMotion input frames running parts of what
|
||||
* VideoMotion does to visualize what it does.
|
||||
* @param {VideoMotion} motion - VideoMotion with inputs to visualize
|
||||
* @param {OUTPUT} output - visualization output mode
|
||||
* @constructor
|
||||
*/
|
||||
class VideoMotionView {
|
||||
constructor (motion, output = OUTPUT.XYT) {
|
||||
/**
|
||||
* VideoMotion instance to visualize.
|
||||
* @type {VideoMotion}
|
||||
*/
|
||||
this.motion = motion;
|
||||
|
||||
/**
|
||||
* Debug canvas to render to.
|
||||
* @type {HTMLCanvasElement}
|
||||
*/
|
||||
const canvas = this.canvas = document.createElement('canvas');
|
||||
canvas.width = WIDTH;
|
||||
canvas.height = HEIGHT;
|
||||
|
||||
/**
|
||||
* 2D context to draw to debug canvas.
|
||||
* @type {CanvasRendering2DContext}
|
||||
*/
|
||||
this.context = canvas.getContext('2d');
|
||||
|
||||
/**
|
||||
* Visualization output mode.
|
||||
* @type {OUTPUT}
|
||||
*/
|
||||
this.output = output;
|
||||
|
||||
/**
|
||||
* Pixel buffer to store output values into before they replace the last frames info in the debug canvas.
|
||||
* @type {Uint32Array}
|
||||
*/
|
||||
this.buffer = new Uint32Array(WIDTH * HEIGHT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Modes of debug output that can be rendered.
|
||||
* @type {object}
|
||||
*/
|
||||
static get OUTPUT () {
|
||||
return OUTPUT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate each pixel address location and call a function with that address.
|
||||
* @param {number} xStart - start location on the x axis of the output pixel buffer
|
||||
* @param {number} yStart - start location on the y axis of the output pixel buffer
|
||||
* @param {nubmer} xStop - location to stop at on the x axis
|
||||
* @param {number} yStop - location to stop at on the y axis
|
||||
* @param {function} fn - handle to call with each iterated address
|
||||
*/
|
||||
_eachAddress (xStart, yStart, xStop, yStop, fn) {
|
||||
for (let i = yStart; i < yStop; i++) {
|
||||
for (let j = xStart; j < xStop; j++) {
|
||||
|
|
@ -45,6 +211,17 @@ class VideoMotionView {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate over cells of pixels and call a function with a function to
|
||||
* iterate over pixel addresses.
|
||||
* @param {number} xStart - start location on the x axis
|
||||
* @param {number} yStart - start lcoation on the y axis
|
||||
* @param {number} xStop - location to stop at on the x axis
|
||||
* @param {number} yStop - location to stop at on the y axis
|
||||
* @param {number} xStep - width of the cells
|
||||
* @param {number} yStep - height of the cells
|
||||
* @param {function} fn - function to call with a bound handle to _eachAddress
|
||||
*/
|
||||
_eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) {
|
||||
const xStep2 = (xStep / 2) | 0;
|
||||
const yStep2 = (yStep / 2) | 0;
|
||||
|
|
@ -61,6 +238,11 @@ class VideoMotionView {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build horizontal, vertical, and temporal difference of a pixel address.
|
||||
* @param {number} address - address to build values for
|
||||
* @returns {object} a object with a gradX, grady, and gradT value
|
||||
*/
|
||||
_grads (address) {
|
||||
const {curr, prev} = this.motion;
|
||||
const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff);
|
||||
|
|
@ -69,6 +251,41 @@ class VideoMotionView {
|
|||
return {gradX, gradY, gradT};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build component values used in determining a motion vector for a pixel
|
||||
* address.
|
||||
* @param {function} eachAddress - a bound handle to _eachAddress to build
|
||||
* component values for
|
||||
* @returns {object} a object with a A2, A1B2, B1, C2, C1 value
|
||||
*/
|
||||
_components (eachAddress) {
|
||||
let A2 = 0;
|
||||
let A1B2 = 0;
|
||||
let B1 = 0;
|
||||
let C2 = 0;
|
||||
let C1 = 0;
|
||||
|
||||
eachAddress(address => {
|
||||
const {gradX, gradY, gradT} = this._grads(address);
|
||||
A2 += gradX * gradX;
|
||||
A1B2 += gradX * gradY;
|
||||
B1 += gradY * gradY;
|
||||
C2 += gradX * gradT;
|
||||
C1 += gradY * gradT;
|
||||
});
|
||||
|
||||
_videoMotionViewComponentsTmp.A2 = A2;
|
||||
_videoMotionViewComponentsTmp.A1B2 = A1B2;
|
||||
_videoMotionViewComponentsTmp.B1 = B1;
|
||||
_videoMotionViewComponentsTmp.C2 = C2;
|
||||
_videoMotionViewComponentsTmp.C1 = C1;
|
||||
return _videoMotionViewComponentsTmp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Visualize the motion code output mode selected for this view to the
|
||||
* debug canvas.
|
||||
*/
|
||||
draw () {
|
||||
if (!(this.motion.prev && this.motion.curr)) {
|
||||
return;
|
||||
|
|
@ -198,16 +415,7 @@ class VideoMotionView {
|
|||
const hmax = HEIGHT - WINSIZE - 1;
|
||||
|
||||
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
||||
let C2 = 0;
|
||||
let C1 = 0;
|
||||
let n = 0;
|
||||
|
||||
eachAddress(address => {
|
||||
const {gradX, gradY, gradT} = this._grads(address);
|
||||
C2 += gradX * gradT;
|
||||
C1 += gradY * gradT;
|
||||
n += 1;
|
||||
});
|
||||
let {C2, C1} = this._components(eachAddress);
|
||||
|
||||
C2 = Math.sqrt(C2);
|
||||
C1 = Math.sqrt(C1);
|
||||
|
|
@ -235,18 +443,7 @@ class VideoMotionView {
|
|||
const hmax = HEIGHT - WINSIZE - 1;
|
||||
|
||||
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
||||
let A2 = 0;
|
||||
let A1B2 = 0;
|
||||
let B1 = 0;
|
||||
let n = 0;
|
||||
|
||||
eachAddress(address => {
|
||||
const {gradX, gradY} = this._grads(address);
|
||||
A2 += gradX * gradX;
|
||||
A1B2 += gradX * gradY;
|
||||
B1 += gradY * gradY;
|
||||
n += 1;
|
||||
});
|
||||
let {A2, A1B2, B1} = this._components(eachAddress);
|
||||
|
||||
A2 = Math.sqrt(A2);
|
||||
A1B2 = Math.sqrt(A1B2);
|
||||
|
|
@ -260,51 +457,38 @@ class VideoMotionView {
|
|||
(A2 & 0xff);
|
||||
});
|
||||
});
|
||||
} else if (this.output === OUTPUT.UV) {
|
||||
const winStep = (WINSIZE * 2) + 1;
|
||||
|
||||
this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
|
||||
const {A2, A1B2, B1, C2, C1} = this._components(fn => fn(address));
|
||||
const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
|
||||
|
||||
const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
|
||||
const hypot = Math.hypot(u, v);
|
||||
const amount = AMOUNT_SCALE * hypot;
|
||||
|
||||
buffer[address] =
|
||||
(0xff << 24) +
|
||||
(inRange && amount > THRESHOLD ?
|
||||
(((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) +
|
||||
(((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) :
|
||||
0x8080
|
||||
);
|
||||
});
|
||||
} else if (this.output === OUTPUT.UV_CELL) {
|
||||
const winStep = (WINSIZE * 2) + 1;
|
||||
const wmax = WIDTH - WINSIZE - 1;
|
||||
const hmax = HEIGHT - WINSIZE - 1;
|
||||
|
||||
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
||||
let A2 = 0;
|
||||
let A1B2 = 0;
|
||||
let B1 = 0;
|
||||
let C2 = 0;
|
||||
let C1 = 0;
|
||||
|
||||
eachAddress(address => {
|
||||
const {gradX, gradY, gradT} = this._grads(address);
|
||||
A2 += gradX * gradX;
|
||||
A1B2 += gradX * gradY;
|
||||
B1 += gradY * gradY;
|
||||
C2 += gradX * gradT;
|
||||
C1 += gradY * gradT;
|
||||
});
|
||||
|
||||
const delta = ((A1B2 * A1B2) - (A2 * B1));
|
||||
let u = 0;
|
||||
let v = 0;
|
||||
if (delta) {
|
||||
/* system is not singular - solving by Kramer method */
|
||||
const deltaX = -((C1 * A1B2) - (C2 * B1));
|
||||
const deltaY = -((A1B2 * C2) - (A2 * C1));
|
||||
const Idelta = 8 / delta;
|
||||
u = deltaX * Idelta;
|
||||
v = deltaY * Idelta;
|
||||
} else {
|
||||
/* singular system - find optical flow in gradient direction */
|
||||
const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
|
||||
if (Norm) {
|
||||
const IGradNorm = 8 / Norm;
|
||||
const temp = -(C1 + C2) * IGradNorm;
|
||||
u = (A1B2 + A2) * temp;
|
||||
v = (B1 + A1B2) * temp;
|
||||
}
|
||||
}
|
||||
const {A2, A1B2, B1, C2, C1} = this._components(eachAddress);
|
||||
const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
|
||||
|
||||
const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
|
||||
const hypot = Math.hypot(u, v);
|
||||
const amount = AMOUNT_SCALE * hypot;
|
||||
|
||||
eachAddress(address => {
|
||||
buffer[address] =
|
||||
(0xff << 24) +
|
||||
|
|
|
|||
Loading…
Reference in a new issue