diff --git a/package.json b/package.json index cc35511c6..b362696e3 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "lodash.defaultsdeep": "4.6.0", "minilog": "3.1.0", "nets": "3.2.0", + "pngjs": "^3.3.2", "promise": "8.0.1", "scratch-audio": "latest", "scratch-blocks": "latest", diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index d794d724b..9b70c498f 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -67,15 +67,18 @@ class Scratch3VideoSensingBlocks { */ this._sampleContext = null; - // Clear target motion state values when the project starts. - this.runtime.on(Runtime.PROJECT_RUN_START, this.reset.bind(this)); + if (this.runtime.ioDevices) { + // Clear target motion state values when the project starts. + this.runtime.on(Runtime.PROJECT_RUN_START, this.reset.bind(this)); - // Boot up the video, canvas to down/up sample the video stream, the - // preview skin and drawable, and kick off looping the analysis logic. - this._setupVideo(); - this._setupSampleCanvas(); - this._setupPreview(); - this._loop(); + // Boot up the video, canvas to down/up sample the video stream, the + // preview skin and drawable, and kick off looping the analysis + // logic. + this._setupVideo(); + this._setupSampleCanvas(); + this._setupPreview(); + this._loop(); + } } /** @@ -317,6 +320,14 @@ class Scratch3VideoSensingBlocks { ]; } + static get MOTION () { + return 1; + } + + static get DIRECTION () { + return 2; + } + /** * An array of info about each drum. * @type {object[]} an array of objects. @@ -336,6 +347,14 @@ class Scratch3VideoSensingBlocks { ]; } + static get STAGE () { + return 1; + } + + static get SPRITE () { + return 2; + } + /** * @returns {object} metadata for this extension and its blocks. */ @@ -352,12 +371,12 @@ class Scratch3VideoSensingBlocks { MOTION_DIRECTION: { type: ArgumentType.NUMBER, menu: 'MOTION_DIRECTION', - defaultValue: 1 + defaultValue: Scratch3VideoSensingBlocks.MOTION }, STAGE_SPRITE: { type: ArgumentType.NUMBER, menu: 'STAGE_SPRITE', - defaultValue: 1 + defaultValue: Scratch3VideoSensingBlocks.STAGE } } }, @@ -406,11 +425,11 @@ class Scratch3VideoSensingBlocks { this.detect.analyzeFrame(); let state = this.detect; - if (Number(args.STAGE_SPRITE) === 2) { + if (Number(args.STAGE_SPRITE) === Scratch3VideoSensingBlocks.SPRITE) { state = this._analyzeLocalMotion(util.target); } - if (Number(args.MOTION_DIRECTION) === 1) { + if (Number(args.MOTION_DIRECTION) === Scratch3VideoSensingBlocks.MOTION) { return state.motionAmount; } return state.motionDirection; diff --git a/test/unit/extension_video_sensing.js b/test/unit/extension_video_sensing.js new file mode 100644 index 000000000..35f0381fa --- /dev/null +++ b/test/unit/extension_video_sensing.js @@ -0,0 +1,409 @@ +const {createReadStream} = require('fs'); +const {join} = require('path'); + +const {PNG} = require('pngjs'); +const {test} = require('tap'); + +const {wrapClamp} = require('../../src/util/math-util'); + +const VideoSensing = require('../../src/extensions/scratch3_video_sensing/index.js'); +const VideoMotion = require('../../src/extensions/scratch3_video_sensing/library.js'); + +/** + * Prefix to the mock frame images used to test the video sensing extension. + * @type {string} + */ +const pngPrefix = 'extension_video_sensing_'; + +/** + * Map of frame keys to the image filenames appended to the pngPrefix. + * @type {object} + */ +const framesMap = { + center: 'center', + left: 'left-5', + left2: 'left-10', + down: 'down-10' +}; + +/** + * Asynchronously read a png file and copy its pixel data into a typed array + * VideoMotion will accept. + * @param {string} name - partial filename to read + * @returns {Promise.} pixel data of the image + */ +const readPNG = name => ( + new Promise((resolve, reject) => { + const png = new PNG(); + createReadStream(join(__dirname, `${pngPrefix}${name}.png`)) + .pipe(png) + .on('parsed', () => { + // Copy the RGBA pixel values into a separate typed array and + // cast the array to Uint32, the array format VideoMotion takes. + resolve(new Uint32Array(new Uint8ClampedArray(png.data).buffer)); + }) + .on('error', reject); + }) +); + +/** + * Read all the frames for testing asynchrnously and produce an object with + * keys following the keys in framesMap. + * @returns {object} mapping of keys in framesMap to image data read from disk + */ +const readFrames = (() => { + // Use this immediately invoking function expression (IIFE) to delay reading + // once to the first test that calls readFrames. + let _promise = null; + + return () => { + if (_promise === null) { + _promise = Promise.all(Object.keys(framesMap).map(key => readPNG(framesMap[key]))) + .then(pngs => ( + Object.keys(framesMap).reduce((frames, key, i) => { + frames[key] = pngs[i]; + return frames; + }, {}) + )); + } + return _promise; + }; +})(); + +/** + * Match if actual is within optMargin to expect. If actual is under -180, + * match if actual + 360 is near expect. If actual is over 180, match if actual + * - 360 is near expect. + * @param {number} actual - actual angle in degrees + * @param {number} expect - expected angle in degrees + * @param {number} optMargin - allowed margin between actual and expect in degrees + * @returns {boolean} true if actual is close to expect + */ +const isNearAngle = (actual, expect, optMargin = 10) => ( + (wrapClamp(actual - expect, 0, 359) < optMargin) || + (wrapClamp(actual - expect, 0, 359) > 360 - optMargin) +); + +// A fake scratch-render drawable that will be used by VideoMotion to restrain +// the area considered for motion detection in VideoMotion.getLocalMotion +const fakeDrawable = { + getFastBounds () { + return { + left: -120, + top: 60, + right: 0, + bottom: -60 + }; + }, + + isTouching () { + return true; + } +}; + +// A fake MotionState used to test the stored values in +// VideoMotion.getLocalMotion, VideoSensing.videoOn and +// VideoSensing.whenMotionGreaterThan. +const fakeMotionState = { + motionFrameNumber: -1, + motionAmount: -1, + motionDirection: -Infinity +}; + +// A fake target referring to the fake drawable and MotionState. +const fakeTarget = { + drawableID: 0, + + getCustomState () { + return fakeMotionState; + }, + setCustomState () {} +}; + +const fakeRuntime = { + targets: [fakeTarget], + + // Without defined devices, VideoSensing will not try to start sampling from + // a video source. + ioDevices: null, + + renderer: { + _allDrawables: [ + fakeDrawable + ] + } +}; + +const fakeBlockUtility = { + target: fakeTarget +}; + +test('detect motionAmount between frames', t => { + t.plan(6); + + return readFrames() + .then(frames => { + const detect = new VideoMotion(); + + // Each of these pairs should have enough motion for the detector. + const framePairs = [ + [frames.center, frames.left], + [frames.center, frames.left2], + [frames.left, frames.left2], + [frames.left, frames.center], + [frames.center, frames.down], + [frames.down, frames.center] + ]; + + // Add both frames of a pair and test for motion. + let index = 0; + for (const [frame1, frame2] of framePairs) { + detect.addFrame(frame1); + detect.addFrame(frame2); + + detect.analyzeFrame(); + t.ok( + detect.motionAmount > 10, + `frame pair ${index + 1} has motion ${detect.motionAmount} over threshold (10)` + ); + index += 1; + } + + t.end(); + }); +}); + +test('detect local motionAmount between frames', t => { + t.plan(6); + + return readFrames() + .then(frames => { + const detect = new VideoMotion(); + + // Each of these pairs should have enough motion for the detector. + const framePairs = [ + [frames.center, frames.left], + [frames.center, frames.left2], + [frames.left, frames.left2], + [frames.left, frames.center], + [frames.center, frames.down], + [frames.down, frames.center] + ]; + + // Add both frames of a pair and test for local motion. + let index = 0; + for (const [frame1, frame2] of framePairs) { + detect.addFrame(frame1); + detect.addFrame(frame2); + + detect.analyzeFrame(); + detect.getLocalMotion(fakeDrawable, fakeMotionState); + t.ok( + fakeMotionState.motionAmount > 10, + `frame pair ${index + 1} has motion ${fakeMotionState.motionAmount} over threshold (10)` + ); + index += 1; + } + + t.end(); + }); +}); + +test('detect motionDirection between frames', t => { + t.plan(6); + + return readFrames() + .then(frames => { + const detect = new VideoMotion(); + + // Each of these pairs is moving in the given direction. Does the detector + // guess a value to that? + const directionMargin = 10; + const framePairs = [ + { + frames: [frames.center, frames.left], + direction: -90 + }, + { + frames: [frames.center, frames.left2], + direction: -90 + }, + { + frames: [frames.left, frames.left2], + direction: -90 + }, + { + frames: [frames.left, frames.center], + direction: 90 + }, + { + frames: [frames.center, frames.down], + direction: 180 + }, + { + frames: [frames.down, frames.center], + direction: 0 + } + ]; + + // Add both frames of a pair and check if the motionDirection is near the + // expected angle. + let index = 0; + for (const {frames: [frame1, frame2], direction} of framePairs) { + detect.addFrame(frame1); + detect.addFrame(frame2); + + detect.analyzeFrame(); + t.ok( + isNearAngle(detect.motionDirection, direction, directionMargin), + `frame pair ${index + 1} is ${detect.motionDirection.toFixed(0)} ` + + `degrees and close to ${direction} degrees` + ); + index += 1; + } + + t.end(); + }); +}); + +test('detect local motionDirection between frames', t => { + t.plan(6); + + return readFrames() + .then(frames => { + const detect = new VideoMotion(); + + // Each of these pairs is moving in the given direction. Does the detector + // guess a value to that? + const directionMargin = 10; + const framePairs = [ + { + frames: [frames.center, frames.left], + direction: -90 + }, + { + frames: [frames.center, frames.left2], + direction: -90 + }, + { + frames: [frames.left, frames.left2], + direction: -90 + }, + { + frames: [frames.left, frames.center], + direction: 90 + }, + { + frames: [frames.center, frames.down], + direction: 180 + }, + { + frames: [frames.down, frames.center], + direction: 0 + } + ]; + + // Add both frames of a pair and check if the local motionDirection is near + // the expected angle. + let index = 0; + for (const {frames: [frame1, frame2], direction} of framePairs) { + detect.addFrame(frame1); + detect.addFrame(frame2); + + detect.analyzeFrame(); + detect.getLocalMotion(fakeDrawable, fakeMotionState); + const motionDirection = fakeMotionState.motionDirection; + t.ok( + isNearAngle(motionDirection, direction, directionMargin), + `frame pair ${index + 1} is ${motionDirection.toFixed(0)} degrees and close to ${direction} degrees` + ); + index += 1; + } + + t.end(); + }); +}); + +test('videoOn returns value dependent on arguments', t => { + t.plan(4); + + return readFrames() + .then(frames => { + const sensing = new VideoSensing(fakeRuntime); + + // With these two frame test if we get expected values depending on the + // arguments to videoOn. + sensing.detect.addFrame(frames.center); + sensing.detect.addFrame(frames.left); + + const motionAmount = sensing.videoOn({ + MOTION_DIRECTION: VideoSensing.MOTION, + STAGE_SPRITE: VideoSensing.STAGE + }, fakeBlockUtility); + t.ok( + motionAmount > 10, + `stage motionAmount ${motionAmount} is over the threshold (10)` + ); + + const localMotionAmount = sensing.videoOn({ + MOTION_DIRECTION: VideoSensing.MOTION, + STAGE_SPRITE: VideoSensing.SPRITE + }, fakeBlockUtility); + t.ok( + localMotionAmount > 10, + `sprite motionAmount ${localMotionAmount} is over the threshold (10)` + ); + + const motionDirection = sensing.videoOn({ + MOTION_DIRECTION: VideoSensing.DIRECTION, + STAGE_SPRITE: VideoSensing.STAGE + }, fakeBlockUtility); + t.ok( + isNearAngle(motionDirection, -90), + `stage motionDirection ${motionDirection.toFixed(0)} degrees is close to ${90} degrees` + ); + + const localMotionDirection = sensing.videoOn({ + MOTION_DIRECTION: VideoSensing.DIRECTION, + STAGE_SPRITE: VideoSensing.SPRITE + }, fakeBlockUtility); + t.ok( + isNearAngle(localMotionDirection, -90), + `sprite motionDirection ${localMotionDirection.toFixed(0)} degrees is close to ${90} degrees` + ); + + t.end(); + }); +}); + +test('whenMotionGreaterThan returns true if local motion meets target', t => { + t.plan(2); + + return readFrames() + .then(frames => { + const sensing = new VideoSensing(fakeRuntime); + + // With these two frame test if we get expected values depending on the + // arguments to whenMotionGreaterThan. + sensing.detect.addFrame(frames.center); + sensing.detect.addFrame(frames.left); + + const over20 = sensing.whenMotionGreaterThan({ + REFERENCE: 20 + }, fakeBlockUtility); + t.ok( + over20, + `enough motion in drawable bounds to reach reference of 20` + ); + + const over80 = sensing.whenMotionGreaterThan({ + REFERENCE: 80 + }, fakeBlockUtility); + t.notOk( + over80, + `not enough motion in drawable bounds to reach reference of 80` + ); + + t.end(); + }); +}); diff --git a/test/unit/extension_video_sensing_center.png b/test/unit/extension_video_sensing_center.png new file mode 100644 index 000000000..eed3cb73e Binary files /dev/null and b/test/unit/extension_video_sensing_center.png differ diff --git a/test/unit/extension_video_sensing_down-10.png b/test/unit/extension_video_sensing_down-10.png new file mode 100644 index 000000000..109d6b8ca Binary files /dev/null and b/test/unit/extension_video_sensing_down-10.png differ diff --git a/test/unit/extension_video_sensing_left-10.png b/test/unit/extension_video_sensing_left-10.png new file mode 100644 index 000000000..8a42a119b Binary files /dev/null and b/test/unit/extension_video_sensing_left-10.png differ diff --git a/test/unit/extension_video_sensing_left-5.png b/test/unit/extension_video_sensing_left-5.png new file mode 100644 index 000000000..5e84c3edf Binary files /dev/null and b/test/unit/extension_video_sensing_left-5.png differ