scratchfoundation/scratch-vm
1
0
Fork 0
mirror of https://github.com/scratchfoundation/scratch-vm.git synced 2025-01-08 14:01:58 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #1007 from mzgoddard/motion-detect-2

Browse source 
Motion detect 2
This commit is contained in:
Ray Schamp 2018-04-05 11:59:59 -04:00 committed by GitHub
commit 41955913aa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 953 additions and 372 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/extensions/scratch3_video_sensing/debug.js
View file

@ -1,4 +1,10 @@
const VideoMotion = require('./lib');
/**
* A debug "index" module exporting VideoMotion and VideoMotionView to debug
* VideoMotion directly.
* @file debug.js
*/
const VideoMotion = require('./library');
const VideoMotionView = require('./view');
module.exports = {

236
src/extensions/scratch3_video_sensing/index.js
View file

@ -1,10 +1,11 @@
const Runtime = require('../../engine/runtime');
const ArgumentType = require('../../extension-support/argument-type');
const BlockType = require('../../extension-support/block-type');
const Clone = require('../../util/clone');
const log = require('../../util/log');
const Timer = require('../../util/timer');
const VideoMotion = require('./lib');
const VideoMotion = require('./library');
/**
* Icon svg to be displayed at the left edge of each extension block, encoded as a data URI.
@ -33,32 +34,123 @@ class Scratch3VideoSensingBlocks {
*/
this.runtime = runtime;
/**
* The motion detection algoritm used to power the motion amount and
* direction values.
* @type {VideoMotion}
*/
this.detect = new VideoMotion();
/**
* The last millisecond epoch timestamp that the video stream was
* analyzed.
* @type {number}
*/
this._lastUpdate = null;
/**
* Id representing a Scratch Renderer skin the video is rendered to for
* previewing.
* @type {number}
*/
this._skinId = -1;
/**
* The Scratch Renderer Skin object.
* @type {Skin}
*/
this._skin = null;
/**
* Id for a drawable using the video's skin that will render as a video
* preview.
* @type {Drawable}
*/
this._drawable = -1;
/**
* Canvas DOM element video is rendered to down or up sample to the
* expected resolution.
* @type {HTMLCanvasElement}
*/
this._sampleCanvas = null;
/**
* Canvas 2D Context to render to the _sampleCanvas member.
* @type {CanvasRenderingContext2D}
*/
this._sampleContext = null;
// Clear target motion state values when the project starts.
this.runtime.on(Runtime.PROJECT_RUN_START, this.reset.bind(this));
// Boot up the video, canvas to down/up sample the video stream, the
// preview skin and drawable, and kick off looping the analysis logic.
this._setupVideo();
this._setupSampleCanvas();
this._setupPreview();
this._loop();
}
static get INTERVAL () {
return 33;
}
/**
* Dimensions the video stream is analyzed at after its rendered to the
* sample canvas.
* @type {Array.<number>}
*/
static get DIMENSIONS () {
return [480, 360];
}
/**
* Order preview drawable is inserted at in the renderer.
* @type {number}
*/
static get ORDER () {
return 1;
}
/**
* The key to load & store a target's motion-related state.
* @type {string}
*/
static get STATE_KEY () {
return 'Scratch.videoSensing';
}
/**
* The default motion-related state, to be used when a target has no existing motion state.
* @type {MotionState}
*/
static get DEFAULT_MOTION_STATE () {
return {
motionFrameNumber: 0,
motionAmount: 0,
motionDirection: 0
};
}
/**
* Reset the extension's data motion detection data. This will clear out
* for example old frames, so the first analyzed frame will not be compared
* against a frame from before reset was called.
*/
reset () {
this.detect.reset();
const targets = this.runtime.targets;
for (let i = 0; i < targets.length; i++) {
const state = targets[i].getCustomState(Scratch3VideoSensingBlocks.STATE_KEY);
if (state) {
state.motionAmount = 0;
state.motionDirection = 0;
}
}
}
/**
* Setup a video element connected to a user media stream.
* @private
*/
_setupVideo () {
this._video = document.createElement('video');
navigator.getUserMedia({
@ -81,6 +173,11 @@ class Scratch3VideoSensingBlocks {
});
}
/**
* Create a campus to render the user media video to down/up sample to the
* needed resolution.
* @private
*/
_setupSampleCanvas () {
// Create low-resolution image to sample video for analysis and preview
const canvas = this._sampleCanvas = document.createElement('canvas');
@ -89,6 +186,11 @@ class Scratch3VideoSensingBlocks {
this._sampleContext = canvas.getContext('2d');
}
/**
* Create a Scratch Renderer Skin and Drawable to preview the user media
* video stream.
* @private
*/
_setupPreview () {
if (this._skinId !== -1) return;
if (this._skin !== null) return;
@ -107,6 +209,11 @@ class Scratch3VideoSensingBlocks {
});
}
/**
* Occasionally step a loop to sample the video, stamp it to the preview
* skin, and add a TypedArray copy of the canvas's pixel data.
* @private
*/
_loop () {
setTimeout(this._loop.bind(this), this.runtime.currentStepTime);
@ -165,9 +272,11 @@ class Scratch3VideoSensingBlocks {
}
/**
* Create data for a menu in scratch-blocks format, consisting of an array of objects with text and
* value properties. The text is a translated string, and the value is one-indexed.
* @param {object[]} info - An array of info objects each having a name property.
* Create data for a menu in scratch-blocks format, consisting of an array
* of objects with text and value properties. The text is a translated
* string, and the value is one-indexed.
* @param {object[]} info - An array of info objects each having a name
* property.
* @return {array} - An array of objects with text and value properties.
* @private
*/
@ -180,27 +289,10 @@ class Scratch3VideoSensingBlocks {
});
}
/**
* The key to load & store a target's motion-related state.
* @type {string}
*/
static get STATE_KEY () {
return 'Scratch.videoSensing';
}
/**
* The default music-related state, to be used when a target has no existing music state.
* @type {MusicState}
*/
static get DEFAULT_MOTION_STATE () {
return {
currentInstrument: 0
};
}
/**
* @param {Target} target - collect motion state for this target.
* @returns {MotionState} the mutable motion state associated with that target. This will be created if necessary.
* @returns {MotionState} the mutable motion state associated with that
* target. This will be created if necessary.
* @private
*/
_getMotionState (target) {
@ -213,7 +305,8 @@ class Scratch3VideoSensingBlocks {
}
/**
* An array of info about each drum.
* An array of choices of whether a reporter should return the frame's
* motion amount or direction.
* @type {object[]} an array of objects.
* @param {string} name - the translatable name to display in the drums menu.
* @param {string} fileName - the name of the audio file containing the drum sound.
@ -232,8 +325,10 @@ class Scratch3VideoSensingBlocks {
/**
* An array of info about each drum.
* @type {object[]} an array of objects.
* @param {string} name - the translatable name to display in the drums menu.
* @param {string} fileName - the name of the audio file containing the drum sound.
* @param {string} name - the translatable name to display in the drums
* menu.
* @param {string} fileName - the name of the audio file containing the
* drum sound.
*/
get STAGE_SPRITE_INFO () {
return [
@ -272,6 +367,19 @@ class Scratch3VideoSensingBlocks {
defaultValue: 1
}
}
},
{
// @todo this hat needs to be set itself to restart existing
// threads like Scratch 2's behaviour.
opcode: 'whenMotionGreaterThan',
text: 'when video motion > [REFERENCE]',
blockType: BlockType.HAT,
arguments: {
REFERENCE: {
type: ArgumentType.NUMBER,
defaultValue: 10
}
}
}
],
menus: {
@ -281,14 +389,32 @@ class Scratch3VideoSensingBlocks {
};
}
/**
* Analyze a part of the frame that a target overlaps.
* @param {Target} target - a target to determine where to analyze
* @returns {MotionState} the motion state for the given target
*/
_analyzeLocalMotion (target) {
const drawable = this.runtime.renderer._allDrawables[target.drawableID];
const state = this._getMotionState(target);
this.detect.getLocalMotion(drawable, state);
return state;
}
/**
* A scratch reporter block handle that analyzes the last two frames and
* depending on the arguments, returns the motion or direction for the
* whole stage or just the target sprite.
* @param {object} args - the block arguments
* @param {BlockUtility} util - the block utility
* @returns {number} the motion amount or direction of the stage or sprite
*/
videoOn (args, util) {
this.detect.analyzeFrame();
let state = this.detect;
if (Number(args.STAGE_SPRITE) === 2) {
const drawable = this.runtime.renderer._allDrawables[util.target.drawableID];
state = this._getMotionState(util.target);
this.detect.getLocalMotion(drawable, state);
state = this._analyzeLocalMotion(util.target);
}
if (Number(args.MOTION_DIRECTION) === 1) {
@ -298,38 +424,18 @@ class Scratch3VideoSensingBlocks {
}
/**
* Check if the stack timer needs initialization.
* @param {object} util - utility object provided by the runtime.
* @return {boolean} - true if the stack timer needs to be initialized.
* @private
* A scratch hat block edge handle that analyzes the last two frames where
* the target sprite overlaps and if it has more motion than the given
* reference value.
* @param {object} args - the block arguments
* @param {BlockUtility} util - the block utility
* @returns {boolean} true if the sprite overlaps more motion than the
* reference
*/
_stackTimerNeedsInit (util) {
return !util.stackFrame.timer;
}
/**
* Start the stack timer and the yield the thread if necessary.
* @param {object} util - utility object provided by the runtime.
* @param {number} duration - a duration in seconds to set the timer for.
* @private
*/
_startStackTimer (util, duration) {
util.stackFrame.timer = new Timer();
util.stackFrame.timer.start();
util.stackFrame.duration = duration;
util.yield();
}
/**
* Check the stack timer, and if its time is not up yet, yield the thread.
* @param {object} util - utility object provided by the runtime.
* @private
*/
_checkStackTimer (util) {
const timeElapsed = util.stackFrame.timer.timeElapsed();
if (timeElapsed < util.stackFrame.duration * 1000) {
util.yield();
}
whenMotionGreaterThan (args, util) {
this.detect.analyzeFrame();
const state = this._analyzeLocalMotion(util.target);
return state.motionAmount > Number(args.REFERENCE);
}
}

248
src/extensions/scratch3_video_sensing/lib.js
View file

@ -1,248 +0,0 @@
/**
* lib.js
*
* Tony Hwang and John Maloney, January 2011
* Michael "Z" Goddard, March 2018
*
* Video motion sensing primitives.
*/
const TO_DEGREE = 180 / Math.PI;
const WIDTH = 480;
const HEIGHT = 360;
// chosen empirically to give a range of roughly 0-100
const AMOUNT_SCALE = 100;
// note 2e-4 * activePixelNum is an experimentally tuned threshold for my
// logitech Pro 9000 webcam - TTH
const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4;
const THRESHOLD = 10;
const WINSIZE = 8;
const LOCAL_MAX_AMOUNT = 100;
const LOCAL_THRESHOLD = THRESHOLD / 3;
const STATE_KEY = 'Scratch.videoSensing';
class VideoMotion {
constructor () {
this.frameNumber = 0;
this.motionAmount = 0;
this.motionDirection = 0;
this.analysisDone = false;
this.curr = null;
this.prev = null;
this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1);
this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT);
this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT);
}
reset () {
this.prev = this.curr = null;
this.motionAmount = this.motionDirection = 0;
this.analysisDone = true;
const targets = this.runtime.targets;
for (let i = 0; i < targets.length; i++) {
targets[i].getCustomState(STATE_KEY).motionAmount = 0;
targets[i].getCustomState(STATE_KEY).motionDirection = 0;
}
}
addFrame (source) {
this.frameNumber++;
this.prev = this.curr;
this.curr = new Uint32Array(source.buffer.slice());
const _tmp = this._prev;
this._prev = this._curr;
this._curr = _tmp;
for (let i = 0; i < this.curr.length; i++) {
this._curr[i] = this.curr[i] & 0xff;
}
this.analysisDone = false;
}
analyzeFrame () {
if (!this.curr || !this.prev) {
this.motionAmount = this.motionDirection = -1;
// don't have two frames to analyze yet
return;
}
const {
_curr: curr,
_prev: prev
} = this;
const winStep = (WINSIZE * 2) + 1;
const wmax = WIDTH - WINSIZE - 1;
const hmax = HEIGHT - WINSIZE - 1;
let uu = 0;
let vv = 0;
let n = 0;
for (let i = WINSIZE + 1; i < hmax; i += winStep) {
for (let j = WINSIZE + 1; j < wmax; j += winStep) {
let A2 = 0;
let A1B2 = 0;
let B1 = 0;
let C1 = 0;
let C2 = 0;
let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE;
let nextAddress = address + winStep;
const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE;
for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) {
for (; address <= nextAddress; address += 1) {
const gradT = ((prev[address]) - (curr[address]));
const gradX = ((curr[address - 1]) - (curr[address + 1]));
const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));
A2 += gradX * gradX;
A1B2 += gradX * gradY;
B1 += gradY * gradY;
C2 += gradX * gradT;
C1 += gradY * gradT;
}
}
const delta = ((A1B2 * A1B2) - (A2 * B1));
let u = 0;
let v = 0;
if (delta) {
// system is not singular - solving by Kramer method
const deltaX = -((C1 * A1B2) - (C2 * B1));
const deltaY = -((A1B2 * C2) - (A2 * C1));
const Idelta = 8 / delta;
u = deltaX * Idelta;
v = deltaY * Idelta;
} else {
// singular system - find optical flow in gradient direction
const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
if (Norm) {
const IGradNorm = 8 / Norm;
const temp = -(C1 + C2) * IGradNorm;
u = (A1B2 + A2) * temp;
v = (B1 + A1B2) * temp;
}
}
if (-winStep < u && u < winStep && -winStep < v && v < winStep) {
uu += u;
vv += v;
n++;
}
}
}
uu /= n;
vv /= n;
this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv));
if (this.motionAmount > THRESHOLD) {
// Scratch direction
this.motionDirection = (((Math.atan2(vv, uu) * TO_DEGREE) + 270) % 360) - 180;
}
this.analysisDone = true;
}
getLocalMotion (drawable, state) {
if (!this.curr || !this.prev) {
state.motionAmount = state.motionDirection = -1;
// don't have two frames to analyze yet
return;
}
if (state.motionFrameNumber !== this.frameNumber) {
const {
_prev: prev,
_curr: curr
} = this;
const boundingRect = drawable.getFastBounds();
const xmin = Math.floor(boundingRect.left + (WIDTH / 2));
const xmax = Math.floor(boundingRect.right + (WIDTH / 2));
const ymin = Math.floor((HEIGHT / 2) - boundingRect.top);
const ymax = Math.floor((HEIGHT / 2) - boundingRect.bottom);
let A2 = 0;
let A1B2 = 0;
let B1 = 0;
let C1 = 0;
let C2 = 0;
let scaleFactor = 0;
const position = [0, 0, 0];
for (let i = ymin; i < ymax; i++) {
for (let j = xmin; j < xmax; j++) {
position[0] = j - (WIDTH / 2);
position[1] = (HEIGHT / 2) - i;
if (
j > 0 && (j < WIDTH - 1) &&
i > 0 && (i < HEIGHT - 1) &&
drawable.isTouching(position)
) {
const address = (i * WIDTH) + j;
const gradT = ((prev[address]) - (curr[address]));
const gradX = ((curr[address - 1]) - (curr[address + 1]));
const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));
A2 += gradX * gradX;
A1B2 += gradX * gradY;
B1 += gradY * gradY;
C2 += gradX * gradT;
C1 += gradY * gradT;
scaleFactor++;
}
}
}
const delta = ((A1B2 * A1B2) - (A2 * B1));
let u = 0;
let v = 0;
if (delta) {
// system is not singular - solving by Kramer method
const deltaX = -((C1 * A1B2) - (C2 * B1));
const deltaY = -((A1B2 * C2) - (A2 * C1));
const Idelta = 8 / delta;
u = deltaX * Idelta;
v = deltaY * Idelta;
} else {
// singular system - find optical flow in gradient direction
const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
if (Norm) {
const IGradNorm = 8 / Norm;
const temp = -(C1 + C2) * IGradNorm;
u = (A1B2 + A2) * temp;
v = (B1 + A1B2) * temp;
}
}
let activePixelNum = 0;
if (scaleFactor) {
// store the area of the sprite in pixels
activePixelNum = scaleFactor;
scaleFactor /= (2 * WINSIZE * 2 * WINSIZE);
u = u / scaleFactor;
v = v / scaleFactor;
}
state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v));
if (state.motionAmount > LOCAL_MAX_AMOUNT) {
// clip all magnitudes greater than 100
state.motionAmount = LOCAL_MAX_AMOUNT;
}
if (state.motionAmount > LOCAL_THRESHOLD) {
// Scratch direction
state.motionDirection = (((Math.atan2(v, u) * TO_DEGREE) + 270) % 360) - 180;
}
state.motionFrameNumber = this.frameNumber;
}
}
}
module.exports = VideoMotion;

380
src/extensions/scratch3_video_sensing/library.js Normal file
View file

@ -0,0 +1,380 @@
/**
* @file library.js
*
* Tony Hwang and John Maloney, January 2011
* Michael "Z" Goddard, March 2018
*
* Video motion sensing primitives.
*/
const {motionVector, scratchAtan2} = require('./math');
/**
* The width of the intended resolution to analyze for motion.
* @type {number}
*/
const WIDTH = 480;
/**
* The height of the intended resolution to analyze for motion.
* @type {number}
*/
const HEIGHT = 360;
/**
* A constant value to scale the magnitude of the x and y components called u
* and v. This creates the motionAmount value.
*
* Old note: chosen empirically to give a range of roughly 0-100
*
* @type {number}
*/
const AMOUNT_SCALE = 100;
/**
* A constant value to scale the magnitude of the x and y components called u
* and v in the local motion derivative. This creates the motionAmount value on
* a target's motion state.
*
* Old note: note 2e-4 * activePixelNum is an experimentally tuned threshold
* for my logitech Pro 9000 webcam - TTH
*
* @type {number}
*/
const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4;
/**
* The motion amount must be higher than the THRESHOLD to calculate a new
* direction value.
* @type {number}
*/
const THRESHOLD = 10;
/**
* The size of the radius of the window of summarized values when considering
* the motion inside the full resolution of the sample.
* @type {number}
*/
const WINSIZE = 8;
/**
* A ceiling for the motionAmount stored to a local target's motion state. The
* motionAmount is not allowed to be larger than LOCAL_MAX_AMOUNT.
* @type {number}
*/
const LOCAL_MAX_AMOUNT = 100;
/**
* The motion amount for a target's local motion must be higher than the
* LOCAL_THRESHOLD to calculate a new direction value.
* @type {number}
*/
const LOCAL_THRESHOLD = THRESHOLD / 3;
/**
* Store the necessary image pixel data to compares frames of a video and
* detect an amount and direction of motion in the full sample or in a
* specified area.
* @constructor
*/
class VideoMotion {
constructor () {
/**
* The number of frames that have been added from a source.
* @type {number}
*/
this.frameNumber = 0;
/**
* The frameNumber last analyzed.
* @type {number}
*/
this.lastAnalyzedFrame = 0;
/**
* The amount of motion detected in the current frame.
* @type {number}
*/
this.motionAmount = 0;
/**
* The direction the motion detected in the frame is general moving in.
* @type {number}
*/
this.motionDirection = 0;
/**
* A copy of the current frame's pixel values. A index of the array is
* represented in RGBA. The lowest byte is red. The next is green. The
* next is blue. And the last is the alpha value of that pixel.
* @type {Uint32Array}
*/
this.curr = null;
/**
* A copy of the last frame's pixel values.
* @type {Uint32Array}
*/
this.prev = null;
/**
* A buffer for holding one component of a pixel's full value twice.
* One for the current value. And one for the last value.
* @type {number}
*/
this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1);
/**
* A clamped uint8 view of _arrays. One component of each index of the
* curr member is copied into this array.
* @type {number}
*/
this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT);
/**
* A clamped uint8 view of _arrays. One component of each index of the
* prev member is copied into this array.
* @type {number}
*/
this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT);
}
/**
* Reset internal state so future frame analysis does not consider values
* from before this method was called.
*/
reset () {
this.frameNumber = 0;
this.lastAnalyzedFrame = 0;
this.motionAmount = this.motionDirection = 0;
this.prev = this.curr = null;
}
/**
* Add a frame to be next analyzed. The passed array represent a pixel with
* each index in the RGBA format.
* @param {Uint32Array} source - a source frame of pixels to copy
*/
addFrame (source) {
this.frameNumber++;
// Swap curr to prev.
this.prev = this.curr;
// Create a clone of the array so any modifications made to the source
// array do not affect the work done in here.
this.curr = new Uint32Array(source.buffer.slice());
// Swap _prev and _curr. Copy one of the color components of the new
// array into _curr overwriting what was the old _prev data.
const _tmp = this._prev;
this._prev = this._curr;
this._curr = _tmp;
for (let i = 0; i < this.curr.length; i++) {
this._curr[i] = this.curr[i] & 0xff;
}
}
/**
* Analyze the current frame against the previous frame determining the
* amount of motion and direction of the motion.
*/
analyzeFrame () {
if (!this.curr || !this.prev) {
this.motionAmount = this.motionDirection = -1;
// Don't have two frames to analyze yet
return;
}
// Return early if new data has not been received.
if (this.lastAnalyzedFrame === this.frameNumber) {
return;
}
this.lastAnalyzedFrame = this.frameNumber;
const {
_curr: curr,
_prev: prev
} = this;
const winStep = (WINSIZE * 2) + 1;
const wmax = WIDTH - WINSIZE - 1;
const hmax = HEIGHT - WINSIZE - 1;
// Accumulate 2d motion vectors from groups of pixels and average it
// later.
let uu = 0;
let vv = 0;
let n = 0;
// Iterate over groups of cells building up the components to determine
// a motion vector for each cell instead of the whole frame to avoid
// integer overflows.
for (let i = WINSIZE + 1; i < hmax; i += winStep) {
for (let j = WINSIZE + 1; j < wmax; j += winStep) {
let A2 = 0;
let A1B2 = 0;
let B1 = 0;
let C1 = 0;
let C2 = 0;
// This is a performance critical math region.
let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE;
let nextAddress = address + winStep;
const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE;
for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) {
for (; address <= nextAddress; address += 1) {
// The difference in color between the last frame and
// the current frame.
const gradT = ((prev[address]) - (curr[address]));
// The difference between the pixel to the left and the
// pixel to the right.
const gradX = ((curr[address - 1]) - (curr[address + 1]));
// The difference between the pixel above and the pixel
// below.
const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));
// Add the combined values of this pixel to previously
// considered pixels.
A2 += gradX * gradX;
A1B2 += gradX * gradY;
B1 += gradY * gradY;
C2 += gradX * gradT;
C1 += gradY * gradT;
}
}
// Use the accumalated values from the for loop to determine a
// motion direction.
const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
// If u and v are within negative winStep to positive winStep,
// add them to a sum that will later be averaged.
if (-winStep < u && u < winStep && -winStep < v && v < winStep) {
uu += u;
vv += v;
n++;
}
}
}
// Average the summed vector values of all of the motion groups.
uu /= n;
vv /= n;
// Scale the magnitude of the averaged UV vector.
this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv));
if (this.motionAmount > THRESHOLD) {
// Scratch direction
this.motionDirection = scratchAtan2(vv, uu);
}
}
/**
* Build motion amount and direction values based on stored current and
* previous frame that overlaps a given drawable.
* @param {Drawable} drawable - touchable and bounded drawable to build motion for
* @param {MotionState} state - state to store built values to
*/
getLocalMotion (drawable, state) {
if (!this.curr || !this.prev) {
state.motionAmount = state.motionDirection = -1;
// Don't have two frames to analyze yet
return;
}
// Skip if the current frame has already been considered for this state.
if (state.motionFrameNumber !== this.frameNumber) {
const {
_prev: prev,
_curr: curr
} = this;
// Restrict the region the amount and direction are built from to
// the area of the current frame overlapped by the given drawable's
// bounding box.
const boundingRect = drawable.getFastBounds();
// Transform the bounding box from scratch space to a space from 0,
// 0 to WIDTH, HEIGHT.
const xmin = Math.max(Math.floor(boundingRect.left + (WIDTH / 2)), 1);
const xmax = Math.min(Math.floor(boundingRect.right + (WIDTH / 2)), WIDTH - 1);
const ymin = Math.max(Math.floor((HEIGHT / 2) - boundingRect.top), 1);
const ymax = Math.min(Math.floor((HEIGHT / 2) - boundingRect.bottom), HEIGHT - 1);
let A2 = 0;
let A1B2 = 0;
let B1 = 0;
let C1 = 0;
let C2 = 0;
let scaleFactor = 0;
const position = [0, 0, 0];
// This is a performance critical math region.
for (let i = ymin; i < ymax; i++) {
for (let j = xmin; j < xmax; j++) {
// i and j are in a coordinate planning ranging from 0 to
// HEIGHT and 0 to WIDTH. Transform that into Scratch's
// range of HEIGHT / 2 to -HEIGHT / 2 and -WIDTH / 2 to
// WIDTH / 2;
position[0] = j - (WIDTH / 2);
position[1] = (HEIGHT / 2) - i;
// Consider only pixels in the drawable that can touch the
// edge or other drawables. Empty space in the current skin
// is skipped.
if (drawable.isTouching(position)) {
const address = (i * WIDTH) + j;
// The difference in color between the last frame and
// the current frame.
const gradT = ((prev[address]) - (curr[address]));
// The difference between the pixel to the left and the
// pixel to the right.
const gradX = ((curr[address - 1]) - (curr[address + 1]));
// The difference between the pixel above and the pixel
// below.
const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));
// Add the combined values of this pixel to previously
// considered pixels.
A2 += gradX * gradX;
A1B2 += gradX * gradY;
B1 += gradY * gradY;
C2 += gradX * gradT;
C1 += gradY * gradT;
scaleFactor++;
}
}
}
// Use the accumalated values from the for loop to determine a
// motion direction.
let {u, v} = motionVector(A2, A1B2, B1, C2, C1);
let activePixelNum = 0;
if (scaleFactor) {
// Store the area of the sprite in pixels
activePixelNum = scaleFactor;
scaleFactor /= (2 * WINSIZE * 2 * WINSIZE);
u = u / scaleFactor;
v = v / scaleFactor;
}
// Scale the magnitude of the averaged UV vector and the number of
// overlapping drawable pixels.
state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v));
if (state.motionAmount > LOCAL_MAX_AMOUNT) {
// Clip all magnitudes greater than 100.
state.motionAmount = LOCAL_MAX_AMOUNT;
}
if (state.motionAmount > LOCAL_THRESHOLD) {
// Scratch direction.
state.motionDirection = scratchAtan2(v, u);
}
// Skip future calls on this state until a new frame is added.
state.motionFrameNumber = this.frameNumber;
}
}
}
module.exports = VideoMotion;

76
src/extensions/scratch3_video_sensing/math.js Normal file
View file

@ -0,0 +1,76 @@
/**
* A constant value helping to transform a value in radians to degrees.
* @type {number}
*/
const TO_DEGREE = 180 / Math.PI;
/**
* A object reused to save on memory allocation returning u and v vector from
* motionVector.
* @type {UV}
*/
const _motionVectorOut = {u: 0, v: 0};
/**
* Determine a motion vector combinations of the color component difference on
* the x axis, y axis, and temporal axis.
* @param {number} A2 - a sum of x axis squared
* @param {number} A1B2 - a sum of x axis times y axis
* @param {number} B1 - a sum of y axis squared
* @param {number} C2 - a sum of x axis times temporal axis
* @param {number} C1 - a sum of y axis times temporal axis
* @param {UV} out - optional object to store return UV info in
* @returns {UV} a uv vector representing the motion for the given input
*/
const motionVector = function (A2, A1B2, B1, C2, C1, out = _motionVectorOut) {
// Compare sums of X * Y and sums of X squared and Y squared.
const delta = ((A1B2 * A1B2) - (A2 * B1));
if (delta) {
// System is not singular - solving by Kramer method.
const deltaX = -((C1 * A1B2) - (C2 * B1));
const deltaY = -((A1B2 * C2) - (A2 * C1));
const Idelta = 8 / delta;
out.u = deltaX * Idelta;
out.v = deltaY * Idelta;
} else {
// Singular system - find optical flow in gradient direction.
const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
if (Norm) {
const IGradNorm = 8 / Norm;
const temp = -(C1 + C2) * IGradNorm;
out.u = (A1B2 + A2) * temp;
out.v = (B1 + A1B2) * temp;
} else {
out.u = 0;
out.v = 0;
}
}
return out;
};
/**
* Translate an angle in degrees with the range -180 to 180 rotated to
* Scratch's reference angle.
* @param {number} degrees - angle in range -180 to 180
* @returns {number} angle from Scratch's reference angle
*/
const scratchDegrees = function (degrees) {
return ((degrees + 270) % 360) - 180;
};
/**
* Get the angle of the y and x component of a 2d vector in degrees in
* Scratch's coordinate plane.
* @param {number} y - the y component of a 2d vector
* @param {number} x - the x component of a 2d vector
* @returns {number} angle in degrees in Scratch's coordinate plane
*/
const scratchAtan2 = function (y, x) {
return scratchDegrees(Math.atan2(y, x) * TO_DEGREE);
};
module.exports = {
motionVector,
scratchDegrees,
scratchAtan2
};

334
src/extensions/scratch3_video_sensing/view.js
View file

@ -1,39 +1,207 @@
const {motionVector} = require('./math');
const WIDTH = 480;
const HEIGHT = 360;
const WINSIZE = 8;
const AMOUNT_SCALE = 100;
const THRESHOLD = 10;
/**
* Modes of debug output that can be rendered.
* @type {object}
*/
const OUTPUT = {
/**
* Render the original input.
* @type {number}
*/
INPUT: -1,
XYT: 0,
XYT_CELL: 1,
XY: 2,
XY_CELL: 3,
/**
* Render the difference of neighboring pixels for each pixel. The
* horizontal difference, or x value, renders in the red output component.
* The vertical difference, or y value, renders in the green output
* component. Pixels with equal neighbors with a kind of lime green or
* #008080 in a RGB hex value. Colors with more red have a lower value to
* the right than the value to the left. Colors with less red have a higher
* value to the right than the value to the left. Similarly colors with
* more green have lower values below than above and colors with less green
* have higher values below than above.
* @type {number}
*/
XY: 0,
/**
* Render the XY output with groups of pixels averaged together. The group
* shape and size matches the full frame's analysis window size.
* @type {number}
*/
XY_CELL: 1,
/**
* Render three color components matching the detection algorith's values
* that multiple the horizontal difference, or x value, and the vertical
* difference, or y value together. The red component is the x value
* squared. The green component is the y value squared. The blue component
* is the x value times the y value. The detection code refers to these
* values as A2, B1, and A1B2.
* @type {number}
*/
AB: 2,
/**
* Render the AB output of groups of pixels summarized by their combined
* square root. The group shape and size matches the full frame's analysis
* window size.
* @type {number}
*/
AB_CELL: 3,
/**
* Render a single color component matching the temporal difference or the
* difference in color for the same pixel coordinate in the current frame
* and the last frame. The difference is rendered in the blue color
* component since x and y axis differences tend to use red and green.
* @type {number}
*/
T: 4,
/**
* Render the T output of groups of pixels averaged. The group shape and
* size matches the full frame's analysis window.
* @type {number}
*/
T_CELL: 5,
C: 6,
AB: 7,
UV: 8
/**
* Render the XY and T outputs together. The x and y axis values use the
* red and green color components as they do in the XY output. The t values
* use the blue color component as the T output does.
* @type {number}
*/
XYT: 6,
/**
* Render the XYT output of groups of pixels averaged. The group shape and
* size matches the full frame's analysis window.
* @type {number}
*/
XYT_CELL: 7,
/**
* Render the horizontal pixel difference times the temporal difference as
* red and the vertical and temporal difference as green. Multiplcation of
* these values ends up with sharp differences in the output showing edge
* details where motion is happening.
* @type {number}
*/
C: 8,
/**
* Render the C output of groups of pixels averaged. The group shape and
* size matches the full frame's analysis window.
* @type {number}
*/
C_CELL: 9,
/**
* Render a per pixel version of UV_CELL. UV_CELL is a close to final step
* of the motion code that builds a motion amount and direction from those
* values. UV_CELL renders grouped summarized values, UV does the per pixel
* version but its can only represent one motion vector code path out of
* two choices. Determining the motion vector compares some of the built
* values but building the values with one pixel ensures this first
* comparison says the values are equal. Even though only one code path is
* used to build the values, its output is close to approximating the
* better solution building vectors from groups of pixels to help
* illustrate when the code determines the motion amount and direction to
* be.
* @type {number}
*/
UV: 10,
/**
* Render cells of mulitple pixels at a step in the motion code that has
* the same cell values and turns them into motion vectors showing the
* amount of motion in the x axis and y axis separately. Those values are a
* step away from becoming a motion amount and direction through standard
* vector to magnitude and angle values.
* @type {number}
*/
UV_CELL: 11
};
/**
* Temporary storage structure for returning values in
* VideoMotionView._components.
* @type {object}
*/
const _videoMotionViewComponentsTmp = {
A2: 0,
A1B2: 0,
B1: 0,
C2: 0,
C1: 0
};
/**
* Manage a debug canvas with VideoMotion input frames running parts of what
* VideoMotion does to visualize what it does.
* @param {VideoMotion} motion - VideoMotion with inputs to visualize
* @param {OUTPUT} output - visualization output mode
* @constructor
*/
class VideoMotionView {
constructor (motion, output = OUTPUT.XYT) {
/**
* VideoMotion instance to visualize.
* @type {VideoMotion}
*/
this.motion = motion;
/**
* Debug canvas to render to.
* @type {HTMLCanvasElement}
*/
const canvas = this.canvas = document.createElement('canvas');
canvas.width = WIDTH;
canvas.height = HEIGHT;
/**
* 2D context to draw to debug canvas.
* @type {CanvasRendering2DContext}
*/
this.context = canvas.getContext('2d');
/**
* Visualization output mode.
* @type {OUTPUT}
*/
this.output = output;
/**
* Pixel buffer to store output values into before they replace the last frames info in the debug canvas.
* @type {Uint32Array}
*/
this.buffer = new Uint32Array(WIDTH * HEIGHT);
}
/**
* Modes of debug output that can be rendered.
* @type {object}
*/
static get OUTPUT () {
return OUTPUT;
}
/**
* Iterate each pixel address location and call a function with that address.
* @param {number} xStart - start location on the x axis of the output pixel buffer
* @param {number} yStart - start location on the y axis of the output pixel buffer
* @param {nubmer} xStop - location to stop at on the x axis
* @param {number} yStop - location to stop at on the y axis
* @param {function} fn - handle to call with each iterated address
*/
_eachAddress (xStart, yStart, xStop, yStop, fn) {
for (let i = yStart; i < yStop; i++) {
for (let j = xStart; j < xStop; j++) {
@ -43,6 +211,17 @@ class VideoMotionView {
}
}
/**
* Iterate over cells of pixels and call a function with a function to
* iterate over pixel addresses.
* @param {number} xStart - start location on the x axis
* @param {number} yStart - start lcoation on the y axis
* @param {number} xStop - location to stop at on the x axis
* @param {number} yStop - location to stop at on the y axis
* @param {number} xStep - width of the cells
* @param {number} yStep - height of the cells
* @param {function} fn - function to call with a bound handle to _eachAddress
*/
_eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) {
const xStep2 = (xStep / 2) | 0;
const yStep2 = (yStep / 2) | 0;
@ -59,6 +238,11 @@ class VideoMotionView {
}
}
/**
* Build horizontal, vertical, and temporal difference of a pixel address.
* @param {number} address - address to build values for
* @returns {object} a object with a gradX, grady, and gradT value
*/
_grads (address) {
const {curr, prev} = this.motion;
const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff);
@ -67,6 +251,41 @@ class VideoMotionView {
return {gradX, gradY, gradT};
}
/**
* Build component values used in determining a motion vector for a pixel
* address.
* @param {function} eachAddress - a bound handle to _eachAddress to build
* component values for
* @returns {object} a object with a A2, A1B2, B1, C2, C1 value
*/
_components (eachAddress) {
let A2 = 0;
let A1B2 = 0;
let B1 = 0;
let C2 = 0;
let C1 = 0;
eachAddress(address => {
const {gradX, gradY, gradT} = this._grads(address);
A2 += gradX * gradX;
A1B2 += gradX * gradY;
B1 += gradY * gradY;
C2 += gradX * gradT;
C1 += gradY * gradT;
});
_videoMotionViewComponentsTmp.A2 = A2;
_videoMotionViewComponentsTmp.A1B2 = A1B2;
_videoMotionViewComponentsTmp.B1 = B1;
_videoMotionViewComponentsTmp.C2 = C2;
_videoMotionViewComponentsTmp.C1 = C1;
return _videoMotionViewComponentsTmp;
}
/**
* Visualize the motion code output mode selected for this view to the
* debug canvas.
*/
draw () {
if (!(this.motion.prev && this.motion.curr)) {
return;
@ -186,63 +405,90 @@ class VideoMotionView {
const {gradX, gradY, gradT} = this._grads(address);
buffer[address] =
(0xff << 24) +
((gradY * gradT) << 8) +
(gradX * gradT);
(((Math.sqrt(gradY * gradT) * 0x0f) & 0xff) << 8) +
((Math.sqrt(gradX * gradT) * 0x0f) & 0xff);
});
}
if (this.output === OUTPUT.C_CELL) {
const winStep = (WINSIZE * 2) + 1;
const wmax = WIDTH - WINSIZE - 1;
const hmax = HEIGHT - WINSIZE - 1;
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
let {C2, C1} = this._components(eachAddress);
C2 = Math.sqrt(C2);
C1 = Math.sqrt(C1);
eachAddress(address => {
buffer[address] =
(0xff << 24) +
((C1 & 0xff) << 8) +
((C2 & 0xff) << 0);
});
});
} else if (this.output === OUTPUT.AB) {
this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
const {gradX, gradY} = this._grads(address);
buffer[address] =
(0xff << 24) +
((gradX * gradY) << 16) +
((gradY * gradY) << 8) +
(gradX * gradX);
(((gradX * gradY) & 0xff) << 16) +
(((gradY * gradY) & 0xff) << 8) +
((gradX * gradX) & 0xff);
});
} else if (this.output === OUTPUT.UV) {
}
if (this.output === OUTPUT.AB_CELL) {
const winStep = (WINSIZE * 2) + 1;
const wmax = WIDTH - WINSIZE - 1;
const hmax = HEIGHT - WINSIZE - 1;
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
let A2 = 0;
let A1B2 = 0;
let B1 = 0;
let C2 = 0;
let C1 = 0;
let {A2, A1B2, B1} = this._components(eachAddress);
A2 = Math.sqrt(A2);
A1B2 = Math.sqrt(A1B2);
B1 = Math.sqrt(B1);
eachAddress(address => {
const {gradX, gradY, gradT} = this._grads(address);
A2 += gradX * gradX;
A1B2 += gradX * gradY;
B1 += gradY * gradY;
C2 += gradX * gradT;
C1 += gradY * gradT;
buffer[address] =
(0xff << 24) +
((A1B2 & 0xff) << 16) +
((B1 & 0xff) << 8) +
(A2 & 0xff);
});
});
} else if (this.output === OUTPUT.UV) {
const winStep = (WINSIZE * 2) + 1;
const delta = ((A1B2 * A1B2) - (A2 * B1));
let u = 0;
let v = 0;
if (delta) {
/* system is not singular - solving by Kramer method */
const deltaX = -((C1 * A1B2) - (C2 * B1));
const deltaY = -((A1B2 * C2) - (A2 * C1));
const Idelta = 8 / delta;
u = deltaX * Idelta;
v = deltaY * Idelta;
} else {
/* singular system - find optical flow in gradient direction */
const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
if (Norm) {
const IGradNorm = 8 / Norm;
const temp = -(C1 + C2) * IGradNorm;
u = (A1B2 + A2) * temp;
v = (B1 + A1B2) * temp;
}
}
this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
const {A2, A1B2, B1, C2, C1} = this._components(fn => fn(address));
const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
const hypot = Math.hypot(u, v);
const amount = AMOUNT_SCALE * hypot;
buffer[address] =
(0xff << 24) +
(inRange && amount > THRESHOLD ?
(((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) +
(((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) :
0x8080
);
});
} else if (this.output === OUTPUT.UV_CELL) {
const winStep = (WINSIZE * 2) + 1;
const wmax = WIDTH - WINSIZE - 1;
const hmax = HEIGHT - WINSIZE - 1;
this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
const {A2, A1B2, B1, C2, C1} = this._components(eachAddress);
const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
const hypot = Math.hypot(u, v);
const amount = AMOUNT_SCALE * hypot;
eachAddress(address => {
buffer[address] =
(0xff << 24) +

4
src/playground/motion.html → src/playground/video-sensing.html
View file

@ -11,8 +11,8 @@
<!-- Stage rendering -->
<script src="./scratch-render.js"></script>
<!-- Extension -->
<script src="./motion-extension.js"></script>
<script src="./video-sensing-extension-debug.js"></script>
<!-- Motion -->
<script src="./motion.js"></script>
<script src="./video-sensing.js"></script>
</body>
</html>

35
src/playground/motion.js → src/playground/video-sensing.js
View file

@ -1,4 +1,7 @@
(function () {
const BENCHMARK_THROTTLE = 250;
const INTERVAL = 33;
const video = document.createElement('video');
navigator.getUserMedia({
audio: false,
@ -17,18 +20,18 @@
video.height = video.videoHeight;
});
}, err => {
/* eslint no-console:0 */
// eslint-disable-next-line no-console
console.log(err);
});
const VideoMotion = window.Scratch3MotionDetect.VideoMotion;
const VideoMotionView = window.Scratch3MotionDetect.VideoMotionView;
const VideoMotion = window.Scratch3VideoSensingDebug.VideoMotion;
const VideoMotionView = window.Scratch3VideoSensingDebug.VideoMotionView;
// Create motion detector
const motion = new VideoMotion();
// Create debug views that will render different slices of how the detector
// uses the a frame of input.
// uses a frame of input.
const OUTPUT = VideoMotionView.OUTPUT;
const outputKeys = Object.keys(OUTPUT);
const outputValues = Object.values(OUTPUT);
@ -36,8 +39,9 @@
.map(output => new VideoMotionView(motion, output));
const view = views[0];
const defaultViews = [OUTPUT.INPUT, OUTPUT.XY_CELL, OUTPUT.T_CELL, OUTPUT.UV];
const defaultViews = [OUTPUT.INPUT, OUTPUT.XY_CELL, OUTPUT.T_CELL, OUTPUT.UV_CELL];
// Add activation toggles for each debug view.
const activators = document.createElement('div');
activators.style.userSelect = 'none';
outputValues.forEach((output, index) => {
@ -66,8 +70,14 @@
// Add a text line to display milliseconds per frame, motion value, and
// motion direction
const textContainer = document.createElement('div');
const textHeader = document.createElement('div');
textHeader.innerText = 'duration (us) :: motion amount :: motion direction';
textContainer.appendChild(textHeader);
const textEl = document.createElement('div');
document.body.appendChild(textEl);
textEl.innerText = `0 :: 0 :: 0`;
textContainer.appendChild(textEl);
document.body.appendChild(textContainer);
let textTimer = Date.now();
// Add the motion debug views to the dom after the text line, so the text
@ -82,7 +92,7 @@
const ctx = tempCanvas.getContext('2d');
const loop = function () {
const timeoutId = setTimeout(loop, 33);
const timeoutId = setTimeout(loop, INTERVAL);
try {
// Get the bitmap data for the video frame
@ -90,15 +100,20 @@
ctx.drawImage(
video,
0, 0, video.width || video.clientWidth, video.height || video.clientHeight,
-480, 0, tempCanvas.width, tempCanvas.height
-tempCanvas.width, 0, tempCanvas.width, tempCanvas.height
);
ctx.resetTransform();
const data = ctx.getImageData(0, 0, tempCanvas.width, tempCanvas.height);
// Analyze the latest frame.
const b = performance.now();
motion.addFrame(data.data);
motion.analyzeFrame();
if (Date.now() - textTimer > 250) {
// Every so often update the visible debug numbers with duration in
// microseconds, the amount of motion and the direction of the
// motion.
if (Date.now() - textTimer > BENCHMARK_THROTTLE) {
const e = performance.now();
const analyzeDuration = ((e - b) * 1000).toFixed(0);
const motionAmount = motion.motionAmount.toFixed(1);
@ -108,7 +123,7 @@
}
views.forEach(_view => _view.active && _view.draw());
} catch (error) {
/* eslint no-console:0 */
// eslint-disable-next-line no-console
console.error(error.stack || error);
clearTimeout(timeoutId);
}

4
webpack.config.js
View file

@ -87,7 +87,7 @@ module.exports = [
// Renderer
'scratch-render'
],
'motion-extension': './src/extensions/scratch3_video_sensing/debug'
'video-sensing-extension-debug': './src/extensions/scratch3_video_sensing/debug'
},
output: {
path: path.resolve(__dirname, 'playground'),
@ -101,7 +101,7 @@ module.exports = [
},
{
test: require.resolve('./src/extensions/scratch3_video_sensing/debug.js'),
loader: 'expose-loader?Scratch3MotionDetect'
loader: 'expose-loader?Scratch3VideoSensingDebug'
},
{
test: require.resolve('stats.js/build/stats.min.js'),