mirror of
https://github.com/scratchfoundation/scratch-vm.git
synced 2024-12-24 06:52:40 -05:00
Document VideoMotionView
This commit is contained in:
parent
9a2e937271
commit
6d2c29530f
1 changed files with 242 additions and 58 deletions
|
@ -1,41 +1,207 @@
|
||||||
|
const {motionVector} = require('./math');
|
||||||
|
|
||||||
const WIDTH = 480;
|
const WIDTH = 480;
|
||||||
const HEIGHT = 360;
|
const HEIGHT = 360;
|
||||||
const WINSIZE = 8;
|
const WINSIZE = 8;
|
||||||
const AMOUNT_SCALE = 100;
|
const AMOUNT_SCALE = 100;
|
||||||
const THRESHOLD = 10;
|
const THRESHOLD = 10;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Modes of debug output that can be rendered.
|
||||||
|
* @type {object}
|
||||||
|
*/
|
||||||
const OUTPUT = {
|
const OUTPUT = {
|
||||||
|
/**
|
||||||
|
* Render the original input.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
INPUT: -1,
|
INPUT: -1,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render the difference of neighboring pixels for each pixel. The
|
||||||
|
* horizontal difference, or x value, renders in the red output component.
|
||||||
|
* The vertical difference, or y value, renders in the green output
|
||||||
|
* component. Pixels with equal neighbors with a kind of lime green or
|
||||||
|
* #008080 in a RGB hex value. Colors with more red have a lower value to
|
||||||
|
* the right than the value to the left. Colors with less red have a higher
|
||||||
|
* value to the right than the value to the left. Similarly colors with
|
||||||
|
* more green have lower values below than above and colors with less green
|
||||||
|
* have higher values below than above.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
XY: 0,
|
XY: 0,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render the XY output with groups of pixels averaged together. The group
|
||||||
|
* shape and size matches the full frame's analysis window size.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
XY_CELL: 1,
|
XY_CELL: 1,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render three color components matching the detection algorith's values
|
||||||
|
* that multiple the horizontal difference, or x value, and the vertical
|
||||||
|
* difference, or y value together. The red component is the x value
|
||||||
|
* squared. The green component is the y value squared. The blue component
|
||||||
|
* is the x value times the y value. The detection code refers to these
|
||||||
|
* values as A2, B1, and A1B2.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
AB: 2,
|
AB: 2,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render the AB output of groups of pixels summarized by their combined
|
||||||
|
* square root. The group shape and size matches the full frame's analysis
|
||||||
|
* window size.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
AB_CELL: 3,
|
AB_CELL: 3,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render a single color component matching the temporal difference or the
|
||||||
|
* difference in color for the same pixel coordinate in the current frame
|
||||||
|
* and the last frame. The difference is rendered in the blue color
|
||||||
|
* component since x and y axis differences tend to use red and green.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
T: 4,
|
T: 4,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render the T output of groups of pixels averaged. The group shape and
|
||||||
|
* size matches the full frame's analysis window.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
T_CELL: 5,
|
T_CELL: 5,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render the XY and T outputs together. The x and y axis values use the
|
||||||
|
* red and green color components as they do in the XY output. The t values
|
||||||
|
* use the blue color component as the T output does.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
XYT: 6,
|
XYT: 6,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render the XYT output of groups of pixels averaged. The group shape and
|
||||||
|
* size matches the full frame's analysis window.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
XYT_CELL: 7,
|
XYT_CELL: 7,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render the horizontal pixel difference times the temporal difference as
|
||||||
|
* red and the vertical and temporal difference as green. Multiplcation of
|
||||||
|
* these values ends up with sharp differences in the output showing edge
|
||||||
|
* details where motion is happening.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
C: 8,
|
C: 8,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render the C output of groups of pixels averaged. The group shape and
|
||||||
|
* size matches the full frame's analysis window.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
C_CELL: 9,
|
C_CELL: 9,
|
||||||
UV_CELL: 10
|
|
||||||
|
/**
|
||||||
|
* Render a per pixel version of UV_CELL. UV_CELL is a close to final step
|
||||||
|
* of the motion code that builds a motion amount and direction from those
|
||||||
|
* values. UV_CELL renders grouped summarized values, UV does the per pixel
|
||||||
|
* version but its can only represent one motion vector code path out of
|
||||||
|
* two choices. Determining the motion vector compares some of the built
|
||||||
|
* values but building the values with one pixel ensures this first
|
||||||
|
* comparison says the values are equal. Even though only one code path is
|
||||||
|
* used to build the values, its output is close to approximating the
|
||||||
|
* better solution building vectors from groups of pixels to help
|
||||||
|
* illustrate when the code determines the motion amount and direction to
|
||||||
|
* be.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
|
UV: 10,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render cells of mulitple pixels at a step in the motion code that has
|
||||||
|
* the same cell values and turns them into motion vectors showing the
|
||||||
|
* amount of motion in the x axis and y axis separately. Those values are a
|
||||||
|
* step away from becoming a motion amount and direction through standard
|
||||||
|
* vector to magnitude and angle values.
|
||||||
|
* @type {number}
|
||||||
|
*/
|
||||||
|
UV_CELL: 11
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Temporary storage structure for returning values in
|
||||||
|
* VideoMotionView._components.
|
||||||
|
* @type {object}
|
||||||
|
*/
|
||||||
|
const _videoMotionViewComponentsTmp = {
|
||||||
|
A2: 0,
|
||||||
|
A1B2: 0,
|
||||||
|
B1: 0,
|
||||||
|
C2: 0,
|
||||||
|
C1: 0
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manage a debug canvas with VideoMotion input frames running parts of what
|
||||||
|
* VideoMotion does to visualize what it does.
|
||||||
|
* @param {VideoMotion} motion - VideoMotion with inputs to visualize
|
||||||
|
* @param {OUTPUT} output - visualization output mode
|
||||||
|
* @constructor
|
||||||
|
*/
|
||||||
class VideoMotionView {
|
class VideoMotionView {
|
||||||
constructor (motion, output = OUTPUT.XYT) {
|
constructor (motion, output = OUTPUT.XYT) {
|
||||||
|
/**
|
||||||
|
* VideoMotion instance to visualize.
|
||||||
|
* @type {VideoMotion}
|
||||||
|
*/
|
||||||
this.motion = motion;
|
this.motion = motion;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Debug canvas to render to.
|
||||||
|
* @type {HTMLCanvasElement}
|
||||||
|
*/
|
||||||
const canvas = this.canvas = document.createElement('canvas');
|
const canvas = this.canvas = document.createElement('canvas');
|
||||||
canvas.width = WIDTH;
|
canvas.width = WIDTH;
|
||||||
canvas.height = HEIGHT;
|
canvas.height = HEIGHT;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 2D context to draw to debug canvas.
|
||||||
|
* @type {CanvasRendering2DContext}
|
||||||
|
*/
|
||||||
this.context = canvas.getContext('2d');
|
this.context = canvas.getContext('2d');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visualization output mode.
|
||||||
|
* @type {OUTPUT}
|
||||||
|
*/
|
||||||
this.output = output;
|
this.output = output;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pixel buffer to store output values into before they replace the last frames info in the debug canvas.
|
||||||
|
* @type {Uint32Array}
|
||||||
|
*/
|
||||||
this.buffer = new Uint32Array(WIDTH * HEIGHT);
|
this.buffer = new Uint32Array(WIDTH * HEIGHT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Modes of debug output that can be rendered.
|
||||||
|
* @type {object}
|
||||||
|
*/
|
||||||
static get OUTPUT () {
|
static get OUTPUT () {
|
||||||
return OUTPUT;
|
return OUTPUT;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Iterate each pixel address location and call a function with that address.
|
||||||
|
* @param {number} xStart - start location on the x axis of the output pixel buffer
|
||||||
|
* @param {number} yStart - start location on the y axis of the output pixel buffer
|
||||||
|
* @param {nubmer} xStop - location to stop at on the x axis
|
||||||
|
* @param {number} yStop - location to stop at on the y axis
|
||||||
|
* @param {function} fn - handle to call with each iterated address
|
||||||
|
*/
|
||||||
_eachAddress (xStart, yStart, xStop, yStop, fn) {
|
_eachAddress (xStart, yStart, xStop, yStop, fn) {
|
||||||
for (let i = yStart; i < yStop; i++) {
|
for (let i = yStart; i < yStop; i++) {
|
||||||
for (let j = xStart; j < xStop; j++) {
|
for (let j = xStart; j < xStop; j++) {
|
||||||
|
@ -45,6 +211,17 @@ class VideoMotionView {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Iterate over cells of pixels and call a function with a function to
|
||||||
|
* iterate over pixel addresses.
|
||||||
|
* @param {number} xStart - start location on the x axis
|
||||||
|
* @param {number} yStart - start lcoation on the y axis
|
||||||
|
* @param {number} xStop - location to stop at on the x axis
|
||||||
|
* @param {number} yStop - location to stop at on the y axis
|
||||||
|
* @param {number} xStep - width of the cells
|
||||||
|
* @param {number} yStep - height of the cells
|
||||||
|
* @param {function} fn - function to call with a bound handle to _eachAddress
|
||||||
|
*/
|
||||||
_eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) {
|
_eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) {
|
||||||
const xStep2 = (xStep / 2) | 0;
|
const xStep2 = (xStep / 2) | 0;
|
||||||
const yStep2 = (yStep / 2) | 0;
|
const yStep2 = (yStep / 2) | 0;
|
||||||
|
@ -61,6 +238,11 @@ class VideoMotionView {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build horizontal, vertical, and temporal difference of a pixel address.
|
||||||
|
* @param {number} address - address to build values for
|
||||||
|
* @returns {object} a object with a gradX, grady, and gradT value
|
||||||
|
*/
|
||||||
_grads (address) {
|
_grads (address) {
|
||||||
const {curr, prev} = this.motion;
|
const {curr, prev} = this.motion;
|
||||||
const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff);
|
const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff);
|
||||||
|
@ -69,6 +251,41 @@ class VideoMotionView {
|
||||||
return {gradX, gradY, gradT};
|
return {gradX, gradY, gradT};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build component values used in determining a motion vector for a pixel
|
||||||
|
* address.
|
||||||
|
* @param {function} eachAddress - a bound handle to _eachAddress to build
|
||||||
|
* component values for
|
||||||
|
* @returns {object} a object with a A2, A1B2, B1, C2, C1 value
|
||||||
|
*/
|
||||||
|
_components (eachAddress) {
|
||||||
|
let A2 = 0;
|
||||||
|
let A1B2 = 0;
|
||||||
|
let B1 = 0;
|
||||||
|
let C2 = 0;
|
||||||
|
let C1 = 0;
|
||||||
|
|
||||||
|
eachAddress(address => {
|
||||||
|
const {gradX, gradY, gradT} = this._grads(address);
|
||||||
|
A2 += gradX * gradX;
|
||||||
|
A1B2 += gradX * gradY;
|
||||||
|
B1 += gradY * gradY;
|
||||||
|
C2 += gradX * gradT;
|
||||||
|
C1 += gradY * gradT;
|
||||||
|
});
|
||||||
|
|
||||||
|
_videoMotionViewComponentsTmp.A2 = A2;
|
||||||
|
_videoMotionViewComponentsTmp.A1B2 = A1B2;
|
||||||
|
_videoMotionViewComponentsTmp.B1 = B1;
|
||||||
|
_videoMotionViewComponentsTmp.C2 = C2;
|
||||||
|
_videoMotionViewComponentsTmp.C1 = C1;
|
||||||
|
return _videoMotionViewComponentsTmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visualize the motion code output mode selected for this view to the
|
||||||
|
* debug canvas.
|
||||||
|
*/
|
||||||
draw () {
|
draw () {
|
||||||
if (!(this.motion.prev && this.motion.curr)) {
|
if (!(this.motion.prev && this.motion.curr)) {
|
||||||
return;
|
return;
|
||||||
|
@ -198,16 +415,7 @@ class VideoMotionView {
|
||||||
const hmax = HEIGHT - WINSIZE - 1;
|
const hmax = HEIGHT - WINSIZE - 1;
|
||||||
|
|
||||||
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
||||||
let C2 = 0;
|
let {C2, C1} = this._components(eachAddress);
|
||||||
let C1 = 0;
|
|
||||||
let n = 0;
|
|
||||||
|
|
||||||
eachAddress(address => {
|
|
||||||
const {gradX, gradY, gradT} = this._grads(address);
|
|
||||||
C2 += gradX * gradT;
|
|
||||||
C1 += gradY * gradT;
|
|
||||||
n += 1;
|
|
||||||
});
|
|
||||||
|
|
||||||
C2 = Math.sqrt(C2);
|
C2 = Math.sqrt(C2);
|
||||||
C1 = Math.sqrt(C1);
|
C1 = Math.sqrt(C1);
|
||||||
|
@ -235,18 +443,7 @@ class VideoMotionView {
|
||||||
const hmax = HEIGHT - WINSIZE - 1;
|
const hmax = HEIGHT - WINSIZE - 1;
|
||||||
|
|
||||||
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
||||||
let A2 = 0;
|
let {A2, A1B2, B1} = this._components(eachAddress);
|
||||||
let A1B2 = 0;
|
|
||||||
let B1 = 0;
|
|
||||||
let n = 0;
|
|
||||||
|
|
||||||
eachAddress(address => {
|
|
||||||
const {gradX, gradY} = this._grads(address);
|
|
||||||
A2 += gradX * gradX;
|
|
||||||
A1B2 += gradX * gradY;
|
|
||||||
B1 += gradY * gradY;
|
|
||||||
n += 1;
|
|
||||||
});
|
|
||||||
|
|
||||||
A2 = Math.sqrt(A2);
|
A2 = Math.sqrt(A2);
|
||||||
A1B2 = Math.sqrt(A1B2);
|
A1B2 = Math.sqrt(A1B2);
|
||||||
|
@ -260,51 +457,38 @@ class VideoMotionView {
|
||||||
(A2 & 0xff);
|
(A2 & 0xff);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
} else if (this.output === OUTPUT.UV) {
|
||||||
|
const winStep = (WINSIZE * 2) + 1;
|
||||||
|
|
||||||
|
this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
|
||||||
|
const {A2, A1B2, B1, C2, C1} = this._components(fn => fn(address));
|
||||||
|
const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
|
||||||
|
|
||||||
|
const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
|
||||||
|
const hypot = Math.hypot(u, v);
|
||||||
|
const amount = AMOUNT_SCALE * hypot;
|
||||||
|
|
||||||
|
buffer[address] =
|
||||||
|
(0xff << 24) +
|
||||||
|
(inRange && amount > THRESHOLD ?
|
||||||
|
(((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) +
|
||||||
|
(((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) :
|
||||||
|
0x8080
|
||||||
|
);
|
||||||
|
});
|
||||||
} else if (this.output === OUTPUT.UV_CELL) {
|
} else if (this.output === OUTPUT.UV_CELL) {
|
||||||
const winStep = (WINSIZE * 2) + 1;
|
const winStep = (WINSIZE * 2) + 1;
|
||||||
const wmax = WIDTH - WINSIZE - 1;
|
const wmax = WIDTH - WINSIZE - 1;
|
||||||
const hmax = HEIGHT - WINSIZE - 1;
|
const hmax = HEIGHT - WINSIZE - 1;
|
||||||
|
|
||||||
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
|
||||||
let A2 = 0;
|
const {A2, A1B2, B1, C2, C1} = this._components(eachAddress);
|
||||||
let A1B2 = 0;
|
const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
|
||||||
let B1 = 0;
|
|
||||||
let C2 = 0;
|
|
||||||
let C1 = 0;
|
|
||||||
|
|
||||||
eachAddress(address => {
|
|
||||||
const {gradX, gradY, gradT} = this._grads(address);
|
|
||||||
A2 += gradX * gradX;
|
|
||||||
A1B2 += gradX * gradY;
|
|
||||||
B1 += gradY * gradY;
|
|
||||||
C2 += gradX * gradT;
|
|
||||||
C1 += gradY * gradT;
|
|
||||||
});
|
|
||||||
|
|
||||||
const delta = ((A1B2 * A1B2) - (A2 * B1));
|
|
||||||
let u = 0;
|
|
||||||
let v = 0;
|
|
||||||
if (delta) {
|
|
||||||
/* system is not singular - solving by Kramer method */
|
|
||||||
const deltaX = -((C1 * A1B2) - (C2 * B1));
|
|
||||||
const deltaY = -((A1B2 * C2) - (A2 * C1));
|
|
||||||
const Idelta = 8 / delta;
|
|
||||||
u = deltaX * Idelta;
|
|
||||||
v = deltaY * Idelta;
|
|
||||||
} else {
|
|
||||||
/* singular system - find optical flow in gradient direction */
|
|
||||||
const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
|
|
||||||
if (Norm) {
|
|
||||||
const IGradNorm = 8 / Norm;
|
|
||||||
const temp = -(C1 + C2) * IGradNorm;
|
|
||||||
u = (A1B2 + A2) * temp;
|
|
||||||
v = (B1 + A1B2) * temp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
|
const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
|
||||||
const hypot = Math.hypot(u, v);
|
const hypot = Math.hypot(u, v);
|
||||||
const amount = AMOUNT_SCALE * hypot;
|
const amount = AMOUNT_SCALE * hypot;
|
||||||
|
|
||||||
eachAddress(address => {
|
eachAddress(address => {
|
||||||
buffer[address] =
|
buffer[address] =
|
||||||
(0xff << 24) +
|
(0xff << 24) +
|
||||||
|
|
Loading…
Reference in a new issue