From 9a2e9372711da7fe3952de6757e24d0d35a22cbc Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:53:37 -0400 Subject: [PATCH] Document VideoMotion class - Rename video_sensing/lib.js to video_sensing/library.js --- .../scratch3_video_sensing/debug.js | 2 +- .../scratch3_video_sensing/index.js | 2 +- src/extensions/scratch3_video_sensing/lib.js | 248 ------------ .../scratch3_video_sensing/library.js | 380 ++++++++++++++++++ 4 files changed, 382 insertions(+), 250 deletions(-) delete mode 100644 src/extensions/scratch3_video_sensing/lib.js create mode 100644 src/extensions/scratch3_video_sensing/library.js diff --git a/src/extensions/scratch3_video_sensing/debug.js b/src/extensions/scratch3_video_sensing/debug.js index 24603b584..0dd3917ef 100644 --- a/src/extensions/scratch3_video_sensing/debug.js +++ b/src/extensions/scratch3_video_sensing/debug.js @@ -4,7 +4,7 @@ * @file debug.js */ -const VideoMotion = require('./lib'); +const VideoMotion = require('./library'); const VideoMotionView = require('./view'); module.exports = { diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 10658f0c6..a59daf489 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -5,7 +5,7 @@ const BlockType = require('../../extension-support/block-type'); const Clone = require('../../util/clone'); const log = require('../../util/log'); -const VideoMotion = require('./lib'); +const VideoMotion = require('./library'); /** * Icon svg to be displayed at the left edge of each extension block, encoded as a data URI. diff --git a/src/extensions/scratch3_video_sensing/lib.js b/src/extensions/scratch3_video_sensing/lib.js deleted file mode 100644 index 611315e1d..000000000 --- a/src/extensions/scratch3_video_sensing/lib.js +++ /dev/null @@ -1,248 +0,0 @@ -/** - * lib.js - * - * Tony Hwang and John Maloney, January 2011 - * Michael "Z" Goddard, March 2018 - * - * Video motion sensing primitives. - */ - -const TO_DEGREE = 180 / Math.PI; -const WIDTH = 480; -const HEIGHT = 360; -// chosen empirically to give a range of roughly 0-100 -const AMOUNT_SCALE = 100; -// note 2e-4 * activePixelNum is an experimentally tuned threshold for my -// logitech Pro 9000 webcam - TTH -const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4; -const THRESHOLD = 10; -const WINSIZE = 8; -const LOCAL_MAX_AMOUNT = 100; -const LOCAL_THRESHOLD = THRESHOLD / 3; - -const STATE_KEY = 'Scratch.videoSensing'; - -class VideoMotion { - constructor () { - this.frameNumber = 0; - this.motionAmount = 0; - this.motionDirection = 0; - this.analysisDone = false; - - this.curr = null; - this.prev = null; - - this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1); - this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT); - this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT); - } - - reset () { - this.prev = this.curr = null; - this.motionAmount = this.motionDirection = 0; - this.analysisDone = true; - - const targets = this.runtime.targets; - for (let i = 0; i < targets.length; i++) { - targets[i].getCustomState(STATE_KEY).motionAmount = 0; - targets[i].getCustomState(STATE_KEY).motionDirection = 0; - } - } - - addFrame (source) { - this.frameNumber++; - - this.prev = this.curr; - this.curr = new Uint32Array(source.buffer.slice()); - - const _tmp = this._prev; - this._prev = this._curr; - this._curr = _tmp; - for (let i = 0; i < this.curr.length; i++) { - this._curr[i] = this.curr[i] & 0xff; - } - - this.analysisDone = false; - } - - analyzeFrame () { - if (!this.curr || !this.prev) { - this.motionAmount = this.motionDirection = -1; - // don't have two frames to analyze yet - return; - } - - const { - _curr: curr, - _prev: prev - } = this; - - const winStep = (WINSIZE * 2) + 1; - const wmax = WIDTH - WINSIZE - 1; - const hmax = HEIGHT - WINSIZE - 1; - - let uu = 0; - let vv = 0; - let n = 0; - - for (let i = WINSIZE + 1; i < hmax; i += winStep) { - for (let j = WINSIZE + 1; j < wmax; j += winStep) { - let A2 = 0; - let A1B2 = 0; - let B1 = 0; - let C1 = 0; - let C2 = 0; - - let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE; - let nextAddress = address + winStep; - const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE; - for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) { - for (; address <= nextAddress; address += 1) { - const gradT = ((prev[address]) - (curr[address])); - const gradX = ((curr[address - 1]) - (curr[address + 1])); - const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); - - A2 += gradX * gradX; - A1B2 += gradX * gradY; - B1 += gradY * gradY; - C2 += gradX * gradT; - C1 += gradY * gradT; - } - } - - const delta = ((A1B2 * A1B2) - (A2 * B1)); - let u = 0; - let v = 0; - if (delta) { - // system is not singular - solving by Kramer method - const deltaX = -((C1 * A1B2) - (C2 * B1)); - const deltaY = -((A1B2 * C2) - (A2 * C1)); - const Idelta = 8 / delta; - u = deltaX * Idelta; - v = deltaY * Idelta; - } else { - // singular system - find optical flow in gradient direction - const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); - if (Norm) { - const IGradNorm = 8 / Norm; - const temp = -(C1 + C2) * IGradNorm; - u = (A1B2 + A2) * temp; - v = (B1 + A1B2) * temp; - } - } - - if (-winStep < u && u < winStep && -winStep < v && v < winStep) { - uu += u; - vv += v; - n++; - } - } - } - - uu /= n; - vv /= n; - this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv)); - if (this.motionAmount > THRESHOLD) { - // Scratch direction - this.motionDirection = (((Math.atan2(vv, uu) * TO_DEGREE) + 270) % 360) - 180; - } - this.analysisDone = true; - } - - getLocalMotion (drawable, state) { - if (!this.curr || !this.prev) { - state.motionAmount = state.motionDirection = -1; - // don't have two frames to analyze yet - return; - } - if (state.motionFrameNumber !== this.frameNumber) { - const { - _prev: prev, - _curr: curr - } = this; - - const boundingRect = drawable.getFastBounds(); - const xmin = Math.floor(boundingRect.left + (WIDTH / 2)); - const xmax = Math.floor(boundingRect.right + (WIDTH / 2)); - const ymin = Math.floor((HEIGHT / 2) - boundingRect.top); - const ymax = Math.floor((HEIGHT / 2) - boundingRect.bottom); - - let A2 = 0; - let A1B2 = 0; - let B1 = 0; - let C1 = 0; - let C2 = 0; - let scaleFactor = 0; - - const position = [0, 0, 0]; - - for (let i = ymin; i < ymax; i++) { - for (let j = xmin; j < xmax; j++) { - position[0] = j - (WIDTH / 2); - position[1] = (HEIGHT / 2) - i; - if ( - j > 0 && (j < WIDTH - 1) && - i > 0 && (i < HEIGHT - 1) && - drawable.isTouching(position) - ) { - const address = (i * WIDTH) + j; - const gradT = ((prev[address]) - (curr[address])); - const gradX = ((curr[address - 1]) - (curr[address + 1])); - const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); - - A2 += gradX * gradX; - A1B2 += gradX * gradY; - B1 += gradY * gradY; - C2 += gradX * gradT; - C1 += gradY * gradT; - scaleFactor++; - } - } - } - - const delta = ((A1B2 * A1B2) - (A2 * B1)); - let u = 0; - let v = 0; - if (delta) { - // system is not singular - solving by Kramer method - const deltaX = -((C1 * A1B2) - (C2 * B1)); - const deltaY = -((A1B2 * C2) - (A2 * C1)); - const Idelta = 8 / delta; - u = deltaX * Idelta; - v = deltaY * Idelta; - } else { - // singular system - find optical flow in gradient direction - const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); - if (Norm) { - const IGradNorm = 8 / Norm; - const temp = -(C1 + C2) * IGradNorm; - u = (A1B2 + A2) * temp; - v = (B1 + A1B2) * temp; - } - } - - let activePixelNum = 0; - if (scaleFactor) { - // store the area of the sprite in pixels - activePixelNum = scaleFactor; - scaleFactor /= (2 * WINSIZE * 2 * WINSIZE); - - u = u / scaleFactor; - v = v / scaleFactor; - } - - state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v)); - if (state.motionAmount > LOCAL_MAX_AMOUNT) { - // clip all magnitudes greater than 100 - state.motionAmount = LOCAL_MAX_AMOUNT; - } - if (state.motionAmount > LOCAL_THRESHOLD) { - // Scratch direction - state.motionDirection = (((Math.atan2(v, u) * TO_DEGREE) + 270) % 360) - 180; - } - state.motionFrameNumber = this.frameNumber; - } - } -} - -module.exports = VideoMotion; diff --git a/src/extensions/scratch3_video_sensing/library.js b/src/extensions/scratch3_video_sensing/library.js new file mode 100644 index 000000000..51a7deef5 --- /dev/null +++ b/src/extensions/scratch3_video_sensing/library.js @@ -0,0 +1,380 @@ +/** + * @file library.js + * + * Tony Hwang and John Maloney, January 2011 + * Michael "Z" Goddard, March 2018 + * + * Video motion sensing primitives. + */ + +const {motionVector, scratchAtan2} = require('./math'); + +/** + * The width of the intended resolution to analyze for motion. + * @type {number} + */ +const WIDTH = 480; + +/** + * The height of the intended resolution to analyze for motion. + * @type {number} + */ +const HEIGHT = 360; + +/** + * A constant value to scale the magnitude of the x and y components called u + * and v. This creates the motionAmount value. + * + * Old note: chosen empirically to give a range of roughly 0-100 + * + * @type {number} + */ +const AMOUNT_SCALE = 100; + +/** + * A constant value to scale the magnitude of the x and y components called u + * and v in the local motion derivative. This creates the motionAmount value on + * a target's motion state. + * + * Old note: note 2e-4 * activePixelNum is an experimentally tuned threshold + * for my logitech Pro 9000 webcam - TTH + * + * @type {number} + */ +const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4; + +/** + * The motion amount must be higher than the THRESHOLD to calculate a new + * direction value. + * @type {number} + */ +const THRESHOLD = 10; + +/** + * The size of the radius of the window of summarized values when considering + * the motion inside the full resolution of the sample. + * @type {number} + */ +const WINSIZE = 8; + +/** + * A ceiling for the motionAmount stored to a local target's motion state. The + * motionAmount is not allowed to be larger than LOCAL_MAX_AMOUNT. + * @type {number} + */ +const LOCAL_MAX_AMOUNT = 100; + +/** + * The motion amount for a target's local motion must be higher than the + * LOCAL_THRESHOLD to calculate a new direction value. + * @type {number} + */ +const LOCAL_THRESHOLD = THRESHOLD / 3; + +/** + * Store the necessary image pixel data to compares frames of a video and + * detect an amount and direction of motion in the full sample or in a + * specified area. + * @constructor + */ +class VideoMotion { + constructor () { + /** + * The number of frames that have been added from a source. + * @type {number} + */ + this.frameNumber = 0; + + /** + * The frameNumber last analyzed. + * @type {number} + */ + this.lastAnalyzedFrame = 0; + + /** + * The amount of motion detected in the current frame. + * @type {number} + */ + this.motionAmount = 0; + + /** + * The direction the motion detected in the frame is general moving in. + * @type {number} + */ + this.motionDirection = 0; + + /** + * A copy of the current frame's pixel values. A index of the array is + * represented in RGBA. The lowest byte is red. The next is green. The + * next is blue. And the last is the alpha value of that pixel. + * @type {Uint32Array} + */ + this.curr = null; + + /** + * A copy of the last frame's pixel values. + * @type {Uint32Array} + */ + this.prev = null; + + /** + * A buffer for holding one component of a pixel's full value twice. + * One for the current value. And one for the last value. + * @type {number} + */ + this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1); + + /** + * A clamped uint8 view of _arrays. One component of each index of the + * curr member is copied into this array. + * @type {number} + */ + this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT); + + /** + * A clamped uint8 view of _arrays. One component of each index of the + * prev member is copied into this array. + * @type {number} + */ + this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT); + } + + /** + * Reset internal state so future frame analysis does not consider values + * from before this method was called. + */ + reset () { + this.frameNumber = 0; + this.lastAnalyzedFrame = 0; + this.motionAmount = this.motionDirection = 0; + this.prev = this.curr = null; + } + + /** + * Add a frame to be next analyzed. The passed array represent a pixel with + * each index in the RGBA format. + * @param {Uint32Array} source - a source frame of pixels to copy + */ + addFrame (source) { + this.frameNumber++; + + // Swap curr to prev. + this.prev = this.curr; + // Create a clone of the array so any modifications made to the source + // array do not affect the work done in here. + this.curr = new Uint32Array(source.buffer.slice()); + + // Swap _prev and _curr. Copy one of the color components of the new + // array into _curr overwriting what was the old _prev data. + const _tmp = this._prev; + this._prev = this._curr; + this._curr = _tmp; + for (let i = 0; i < this.curr.length; i++) { + this._curr[i] = this.curr[i] & 0xff; + } + } + + /** + * Analyze the current frame against the previous frame determining the + * amount of motion and direction of the motion. + */ + analyzeFrame () { + if (!this.curr || !this.prev) { + this.motionAmount = this.motionDirection = -1; + // Don't have two frames to analyze yet + return; + } + + // Return early if new data has not been received. + if (this.lastAnalyzedFrame === this.frameNumber) { + return; + } + this.lastAnalyzedFrame = this.frameNumber; + + const { + _curr: curr, + _prev: prev + } = this; + + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + // Accumulate 2d motion vectors from groups of pixels and average it + // later. + let uu = 0; + let vv = 0; + let n = 0; + + // Iterate over groups of cells building up the components to determine + // a motion vector for each cell instead of the whole frame to avoid + // integer overflows. + for (let i = WINSIZE + 1; i < hmax; i += winStep) { + for (let j = WINSIZE + 1; j < wmax; j += winStep) { + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C1 = 0; + let C2 = 0; + + // This is a performance critical math region. + let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE; + let nextAddress = address + winStep; + const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE; + for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) { + for (; address <= nextAddress; address += 1) { + // The difference in color between the last frame and + // the current frame. + const gradT = ((prev[address]) - (curr[address])); + // The difference between the pixel to the left and the + // pixel to the right. + const gradX = ((curr[address - 1]) - (curr[address + 1])); + // The difference between the pixel above and the pixel + // below. + const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); + + // Add the combined values of this pixel to previously + // considered pixels. + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + } + } + + // Use the accumalated values from the for loop to determine a + // motion direction. + const {u, v} = motionVector(A2, A1B2, B1, C2, C1); + + // If u and v are within negative winStep to positive winStep, + // add them to a sum that will later be averaged. + if (-winStep < u && u < winStep && -winStep < v && v < winStep) { + uu += u; + vv += v; + n++; + } + } + } + + // Average the summed vector values of all of the motion groups. + uu /= n; + vv /= n; + + // Scale the magnitude of the averaged UV vector. + this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv)); + if (this.motionAmount > THRESHOLD) { + // Scratch direction + this.motionDirection = scratchAtan2(vv, uu); + } + } + + /** + * Build motion amount and direction values based on stored current and + * previous frame that overlaps a given drawable. + * @param {Drawable} drawable - touchable and bounded drawable to build motion for + * @param {MotionState} state - state to store built values to + */ + getLocalMotion (drawable, state) { + if (!this.curr || !this.prev) { + state.motionAmount = state.motionDirection = -1; + // Don't have two frames to analyze yet + return; + } + + // Skip if the current frame has already been considered for this state. + if (state.motionFrameNumber !== this.frameNumber) { + const { + _prev: prev, + _curr: curr + } = this; + + // Restrict the region the amount and direction are built from to + // the area of the current frame overlapped by the given drawable's + // bounding box. + const boundingRect = drawable.getFastBounds(); + // Transform the bounding box from scratch space to a space from 0, + // 0 to WIDTH, HEIGHT. + const xmin = Math.max(Math.floor(boundingRect.left + (WIDTH / 2)), 1); + const xmax = Math.min(Math.floor(boundingRect.right + (WIDTH / 2)), WIDTH - 1); + const ymin = Math.max(Math.floor((HEIGHT / 2) - boundingRect.top), 1); + const ymax = Math.min(Math.floor((HEIGHT / 2) - boundingRect.bottom), HEIGHT - 1); + + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C1 = 0; + let C2 = 0; + let scaleFactor = 0; + + const position = [0, 0, 0]; + + // This is a performance critical math region. + for (let i = ymin; i < ymax; i++) { + for (let j = xmin; j < xmax; j++) { + // i and j are in a coordinate planning ranging from 0 to + // HEIGHT and 0 to WIDTH. Transform that into Scratch's + // range of HEIGHT / 2 to -HEIGHT / 2 and -WIDTH / 2 to + // WIDTH / 2; + position[0] = j - (WIDTH / 2); + position[1] = (HEIGHT / 2) - i; + // Consider only pixels in the drawable that can touch the + // edge or other drawables. Empty space in the current skin + // is skipped. + if (drawable.isTouching(position)) { + const address = (i * WIDTH) + j; + // The difference in color between the last frame and + // the current frame. + const gradT = ((prev[address]) - (curr[address])); + // The difference between the pixel to the left and the + // pixel to the right. + const gradX = ((curr[address - 1]) - (curr[address + 1])); + // The difference between the pixel above and the pixel + // below. + const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); + + // Add the combined values of this pixel to previously + // considered pixels. + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + scaleFactor++; + } + } + } + + // Use the accumalated values from the for loop to determine a + // motion direction. + let {u, v} = motionVector(A2, A1B2, B1, C2, C1); + + let activePixelNum = 0; + if (scaleFactor) { + // Store the area of the sprite in pixels + activePixelNum = scaleFactor; + + scaleFactor /= (2 * WINSIZE * 2 * WINSIZE); + u = u / scaleFactor; + v = v / scaleFactor; + } + + // Scale the magnitude of the averaged UV vector and the number of + // overlapping drawable pixels. + state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v)); + if (state.motionAmount > LOCAL_MAX_AMOUNT) { + // Clip all magnitudes greater than 100. + state.motionAmount = LOCAL_MAX_AMOUNT; + } + if (state.motionAmount > LOCAL_THRESHOLD) { + // Scratch direction. + state.motionDirection = scratchAtan2(v, u); + } + + // Skip future calls on this state until a new frame is added. + state.motionFrameNumber = this.frameNumber; + } + } +} + +module.exports = VideoMotion;