From 3c473f6ba8c62afd3282665b1b87ed719609df55 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Thu, 22 Mar 2018 11:51:44 -0400 Subject: [PATCH 1/3] Translate scratch 2 video sensing into scratch 3 extension - Add videoSensing extension to extension-manager --- src/extension-support/extension-manager.js | 5 +- .../scratch3_video_sensing/debug.js | 7 + .../scratch3_video_sensing/index.js | 336 ++++++++++++++++++ src/extensions/scratch3_video_sensing/lib.js | 248 +++++++++++++ src/extensions/scratch3_video_sensing/view.js | 263 ++++++++++++++ 5 files changed, 858 insertions(+), 1 deletion(-) create mode 100644 src/extensions/scratch3_video_sensing/debug.js create mode 100644 src/extensions/scratch3_video_sensing/index.js create mode 100644 src/extensions/scratch3_video_sensing/lib.js create mode 100644 src/extensions/scratch3_video_sensing/view.js diff --git a/src/extension-support/extension-manager.js b/src/extension-support/extension-manager.js index bf988adb9..689fcf128 100644 --- a/src/extension-support/extension-manager.js +++ b/src/extension-support/extension-manager.js @@ -9,10 +9,13 @@ const BlockType = require('./block-type'); const Scratch3PenBlocks = require('../extensions/scratch3_pen'); const Scratch3WeDo2Blocks = require('../extensions/scratch3_wedo2'); const Scratch3MusicBlocks = require('../extensions/scratch3_music'); +const Scratch3VideoSensingBlocks = require('../extensions/scratch3_video_sensing'); + const builtinExtensions = { pen: Scratch3PenBlocks, wedo2: Scratch3WeDo2Blocks, - music: Scratch3MusicBlocks + music: Scratch3MusicBlocks, + videoSensing: Scratch3VideoSensingBlocks }; /** diff --git a/src/extensions/scratch3_video_sensing/debug.js b/src/extensions/scratch3_video_sensing/debug.js new file mode 100644 index 000000000..a7a304880 --- /dev/null +++ b/src/extensions/scratch3_video_sensing/debug.js @@ -0,0 +1,7 @@ +const VideoMotion = require('./lib'); +const VideoMotionView = require('./view'); + +module.exports = { + VideoMotion, + VideoMotionView +}; diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js new file mode 100644 index 000000000..506d114f5 --- /dev/null +++ b/src/extensions/scratch3_video_sensing/index.js @@ -0,0 +1,336 @@ +const ArgumentType = require('../../extension-support/argument-type'); +const BlockType = require('../../extension-support/block-type'); +const Clone = require('../../util/clone'); +const log = require('../../util/log'); +const Timer = require('../../util/timer'); + +const VideoMotion = require('./lib'); + +/** + * Icon svg to be displayed at the left edge of each extension block, encoded as a data URI. + * @type {string} + */ +// eslint-disable-next-line max-len +const blockIconURI = 'data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNDAiIGhlaWdodD0iNDAiIHZpZXdCb3g9IjAgMCA0MCA0MCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+PHRpdGxlPm11c2ljLWJsb2NrLWljb248L3RpdGxlPjxkZWZzPjxwYXRoIGQ9Ik0zMi4xOCAyNS44NzRDMzIuNjM2IDI4LjE1NyAzMC41MTIgMzAgMjcuNDMzIDMwYy0zLjA3IDAtNS45MjMtMS44NDMtNi4zNzItNC4xMjYtLjQ1OC0yLjI4NSAxLjY2NS00LjEzNiA0Ljc0My00LjEzNi42NDcgMCAxLjI4My4wODQgMS44OS4yMzQuMzM4LjA4Ni42MzcuMTguOTM4LjMwMi44Ny0uMDItLjEwNC0yLjI5NC0xLjgzNS0xMi4yMy0yLjEzNC0xMi4zMDIgMy4wNi0xLjg3IDguNzY4LTIuNzUyIDUuNzA4LS44ODUuMDc2IDQuODItMy42NSAzLjg0NC0zLjcyNC0uOTg3LTQuNjUtNy4xNTMuMjYzIDE0LjczOHptLTE2Ljk5OCA1Ljk5QzE1LjYzIDM0LjE0OCAxMy41MDcgMzYgMTAuNDQgMzZjLTMuMDcgMC01LjkyMi0xLjg1Mi02LjM4LTQuMTM2LS40NDgtMi4yODQgMS42NzQtNC4xMzUgNC43NS00LjEzNSAxLjAwMyAwIDEuOTc1LjE5NiAyLjg1NS41NDMuODIyLS4wNTUtLjE1LTIuMzc3LTEuODYyLTEyLjIyOC0yLjEzMy0xMi4zMDMgMy4wNi0xLjg3IDguNzY0LTIuNzUzIDUuNzA2LS44OTQuMDc2IDQuODItMy42NDggMy44MzQtMy43MjQtLjk4Ny00LjY1LTcuMTUyLjI2MiAxNC43Mzh6IiBpZD0iYSIvPjwvZGVmcz48ZyBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPjx1c2UgZmlsbD0iI0ZGRiIgeGxpbms6aHJlZj0iI2EiLz48cGF0aCBzdHJva2Utb3BhY2l0eT0iLjEiIHN0cm9rZT0iIzAwMCIgZD0iTTI4LjQ1NiAyMS42NzVjLS4wMS0uMzEyLS4wODctLjgyNS0uMjU2LTEuNzAyLS4wOTYtLjQ5NS0uNjEyLTMuMDIyLS43NTMtMy43My0uMzk1LTEuOTgtLjc2LTMuOTItMS4xNDItNi4xMTMtLjczMi00LjIyMy0uNjkzLTYuMDUuMzQ0LTYuNTI3LjUtLjIzIDEuMDYtLjA4IDEuODQuMzUuNDE0LjIyNyAyLjE4MiAxLjM2NSAyLjA3IDEuMjk2IDEuOTk0IDEuMjQyIDMuNDY0IDEuNzc0IDQuOTMgMS41NDggMS41MjYtLjIzNyAyLjUwNC0uMDYgMi44NzYuNjE4LjM0OC42MzUuMDE1IDEuNDE2LS43MyAyLjE4LTEuNDcyIDEuNTE2LTMuOTc1IDIuNTE0LTUuODQ4IDIuMDIzLS44MjItLjIyLTEuMjM4LS40NjUtMi4zOC0xLjI2N2wtLjA5NS0uMDY2Yy4wNDcuNTkzLjI2NCAxLjc0LjcxNyAzLjgwMy4yOTQgMS4zMzYgMi4wOCA5LjE4NyAyLjYzNyAxMS42NzRsLjAwMi4wMTJjLjUyOCAyLjYzNy0xLjg3MyA0LjcyNC01LjIzNiA0LjcyNC0zLjI5IDAtNi4zNjMtMS45ODgtNi44NjItNC41MjgtLjUzLTIuNjQgMS44NzMtNC43MzQgNS4yMzMtNC43MzQuNjcyIDAgMS4zNDcuMDg1IDIuMDE0LjI1LjIyNy4wNTcuNDM2LjExOC42MzYuMTg3em0tMTYuOTk2IDUuOTljLS4wMS0uMzE4LS4wOS0uODM4LS4yNjYtMS43MzctLjA5LS40Ni0uNTk1LTIuOTM3LS43NTMtMy43MjctLjM5LTEuOTYtLjc1LTMuODktMS4xMy02LjA3LS43MzItNC4yMjMtLjY5Mi02LjA1LjM0NC02LjUyNi41MDItLjIzIDEuMDYtLjA4MiAxLjg0LjM1LjQxNS4yMjcgMi4xODIgMS4zNjQgMi4wNyAxLjI5NSAxLjk5MyAxLjI0MiAzLjQ2MiAxLjc3NCA0LjkyNiAxLjU0OCAxLjUyNS0uMjQgMi41MDQtLjA2NCAyLjg3Ni42MTQuMzQ4LjYzNS4wMTUgMS40MTUtLjcyOCAyLjE4LTEuNDc0IDEuNTE3LTMuOTc3IDIuNTEzLTUuODQ3IDIuMDE3LS44Mi0uMjItMS4yMzYtLjQ2NC0yLjM3OC0xLjI2N2wtLjA5NS0uMDY1Yy4wNDcuNTkzLjI2NCAxLjc0LjcxNyAzLjgwMi4yOTQgMS4zMzcgMi4wNzggOS4xOSAyLjYzNiAxMS42NzVsLjAwMy4wMTNjLjUxNyAyLjYzOC0xLjg4NCA0LjczMi01LjIzNCA0LjczMi0zLjI4NyAwLTYuMzYtMS45OTMtNi44Ny00LjU0LS41Mi0yLjY0IDEuODg0LTQuNzMgNS4yNC00LjczLjkwNSAwIDEuODAzLjE1IDIuNjUuNDM2eiIvPjwvZz48L3N2Zz4='; + +/** + * Icon svg to be displayed in the category menu, encoded as a data URI. + * @type {string} + */ +// eslint-disable-next-line max-len +const menuIconURI = 'data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAiIGhlaWdodD0iMjAiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PHBhdGggZD0iTTE2LjA5IDEyLjkzN2MuMjI4IDEuMTQxLS44MzMgMi4wNjMtMi4zNzMgMi4wNjMtMS41MzUgMC0yLjk2Mi0uOTIyLTMuMTg2LTIuMDYzLS4yMy0xLjE0Mi44MzMtMi4wNjggMi4zNzItMi4wNjguMzIzIDAgLjY0MS4wNDIuOTQ1LjExN2EzLjUgMy41IDAgMCAxIC40NjguMTUxYy40MzUtLjAxLS4wNTItMS4xNDctLjkxNy02LjExNC0xLjA2Ny02LjE1MiAxLjUzLS45MzUgNC4zODQtMS4zNzcgMi44NTQtLjQ0Mi4wMzggMi40MS0xLjgyNSAxLjkyMi0xLjg2Mi0uNDkzLTIuMzI1LTMuNTc3LjEzMiA3LjM3ek03LjQ2IDguNTYzYy0xLjg2Mi0uNDkzLTIuMzI1LTMuNTc2LjEzIDcuMzdDNy44MTYgMTcuMDczIDYuNzU0IDE4IDUuMjIgMThjLTEuNTM1IDAtMi45NjEtLjkyNi0zLjE5LTIuMDY4LS4yMjQtMS4xNDIuODM3LTIuMDY3IDIuMzc1LTIuMDY3LjUwMSAwIC45ODcuMDk4IDEuNDI3LjI3Mi40MTItLjAyOC0uMDc0LTEuMTg5LS45My02LjExNEMzLjgzNCAxLjg3IDYuNDMgNy4wODcgOS4yODIgNi42NDZjMi44NTQtLjQ0Ny4wMzggMi40MS0xLjgyMyAxLjkxN3oiIGZpbGw9IiM1NzVFNzUiIGZpbGwtcnVsZT0iZXZlbm9kZCIvPjwvc3ZnPg=='; + +/** + * Class for the motion-related blocks in Scratch 3.0 + * @param {Runtime} runtime - the runtime instantiating this block package. + * @constructor + */ +class Scratch3VideoSensingBlocks { + constructor (runtime) { + /** + * The runtime instantiating this block package. + * @type {Runtime} + */ + this.runtime = runtime; + + this.detect = new VideoMotion(); + + this._lastUpdate = null; + + this._skinId = -1; + this._skin = null; + this._drawable = -1; + + this._setupVideo(); + this._setupSampleCanvas(); + this._setupPreview(); + this._loop(); + } + + static get INTERVAL () { + return 33; + } + + static get DIMENSIONS () { + return [480, 360]; + } + + static get ORDER () { + return 1; + } + + _setupVideo () { + this._video = document.createElement('video'); + navigator.getUserMedia({ + audio: false, + video: { + width: {min: 480, ideal: 640}, + height: {min: 360, ideal: 480} + } + }, stream => { + this._video.src = window.URL.createObjectURL(stream); + // Hint to the stream that it should load. A standard way to do this + // is add the video tag to the DOM. Since this extension wants to + // hide the video tag and instead render a sample of the stream into + // the webgl rendered Scratch canvas, another hint like this one is + // needed. + this._track = stream.getTracks()[0]; + }, err => { + // @todo Properly handle errors + log(err); + }); + } + + _setupSampleCanvas () { + // Create low-resolution image to sample video for analysis and preview + const canvas = this._sampleCanvas = document.createElement('canvas'); + canvas.width = Scratch3VideoSensingBlocks.DIMENSIONS[0]; + canvas.height = Scratch3VideoSensingBlocks.DIMENSIONS[1]; + this._sampleContext = canvas.getContext('2d'); + } + + _setupPreview () { + if (this._skinId !== -1) return; + if (this._skin !== null) return; + if (this._drawable !== -1) return; + if (!this.runtime.renderer) return; + + this._skinId = this.runtime.renderer.createPenSkin(); + this._skin = this.runtime.renderer._allSkins[this._skinId]; + this._drawable = this.runtime.renderer.createDrawable(); + this.runtime.renderer.setDrawableOrder( + this._drawable, + Scratch3VideoSensingBlocks.ORDER + ); + this.runtime.renderer.updateDrawableProperties(this._drawable, { + skinId: this._skinId + }); + } + + _loop () { + setTimeout(this._loop.bind(this), this.runtime.currentStepTime); + + // Ensure video stream is established + if (!this._video) return; + if (!this._track) return; + if (typeof this._video.videoWidth !== 'number') return; + if (typeof this._video.videoHeight !== 'number') return; + + // Bail if the camera is *still* not ready + const nativeWidth = this._video.videoWidth; + const nativeHeight = this._video.videoHeight; + if (nativeWidth === 0) return; + if (nativeHeight === 0) return; + + const ctx = this._sampleContext; + + // Mirror + ctx.scale(-1, 1); + + // Generate video thumbnail for analysis + ctx.drawImage( + this._video, + 0, + 0, + nativeWidth, + nativeHeight, + Scratch3VideoSensingBlocks.DIMENSIONS[0] * -1, + 0, + Scratch3VideoSensingBlocks.DIMENSIONS[0], + Scratch3VideoSensingBlocks.DIMENSIONS[1] + ); + + // Restore the canvas transform + ctx.resetTransform(); + + // Render to preview layer + if (this._skin !== null) { + const xOffset = Scratch3VideoSensingBlocks.DIMENSIONS[0] / 2 * -1; + const yOffset = Scratch3VideoSensingBlocks.DIMENSIONS[1] / 2; + this._skin.drawStamp(this._sampleCanvas, xOffset, yOffset); + this.runtime.requestRedraw(); + } + + // Add frame to detector + const time = Date.now(); + if (this._lastUpdate === null) this._lastUpdate = time; + const offset = time - this._lastUpdate; + if (offset > Scratch3VideoSensingBlocks.INTERVAL) { + this._lastUpdate = time; + const data = ctx.getImageData( + 0, 0, Scratch3VideoSensingBlocks.DIMENSIONS[0], Scratch3VideoSensingBlocks.DIMENSIONS[1] + ); + this.detect.addFrame(data.data); + } + } + + /** + * Create data for a menu in scratch-blocks format, consisting of an array of objects with text and + * value properties. The text is a translated string, and the value is one-indexed. + * @param {object[]} info - An array of info objects each having a name property. + * @return {array} - An array of objects with text and value properties. + * @private + */ + _buildMenu (info) { + return info.map((entry, index) => { + const obj = {}; + obj.text = entry.name; + obj.value = String(index + 1); + return obj; + }); + } + + /** + * The key to load & store a target's motion-related state. + * @type {string} + */ + static get STATE_KEY () { + return 'Scratch.videoSensing'; + } + + /** + * The default music-related state, to be used when a target has no existing music state. + * @type {MusicState} + */ + static get DEFAULT_MOTION_STATE () { + return { + currentInstrument: 0 + }; + } + + /** + * @param {Target} target - collect motion state for this target. + * @returns {MotionState} the mutable motion state associated with that target. This will be created if necessary. + * @private + */ + _getMotionState (target) { + let motionState = target.getCustomState(Scratch3VideoSensingBlocks.STATE_KEY); + if (!motionState) { + motionState = Clone.simple(Scratch3VideoSensingBlocks.DEFAULT_MOTION_STATE); + target.setCustomState(Scratch3VideoSensingBlocks.STATE_KEY, motionState); + } + return motionState; + } + + /** + * An array of info about each drum. + * @type {object[]} an array of objects. + * @param {string} name - the translatable name to display in the drums menu. + * @param {string} fileName - the name of the audio file containing the drum sound. + */ + get MOTION_DIRECTION_INFO () { + return [ + { + name: 'motion' + }, + { + name: 'direction' + } + ]; + } + + /** + * An array of info about each drum. + * @type {object[]} an array of objects. + * @param {string} name - the translatable name to display in the drums menu. + * @param {string} fileName - the name of the audio file containing the drum sound. + */ + get STAGE_SPRITE_INFO () { + return [ + { + name: 'stage' + }, + { + name: 'sprite' + } + ]; + } + + /** + * @returns {object} metadata for this extension and its blocks. + */ + getInfo () { + return { + id: 'videoSensing', + name: 'Video Sensing', + menuIconURI: menuIconURI, + blockIconURI: blockIconURI, + blocks: [ + { + opcode: 'videoOn', + blockType: BlockType.REPORTER, + text: 'video [MOTION_DIRECTION] on [STAGE_SPRITE]', + arguments: { + MOTION_DIRECTION: { + type: ArgumentType.NUMBER, + menu: 'MOTION_DIRECTION', + defaultValue: 1 + }, + STAGE_SPRITE: { + type: ArgumentType.NUMBER, + menu: 'STAGE_SPRITE', + defaultValue: 1 + } + } + } + ], + menus: { + MOTION_DIRECTION: this._buildMenu(this.MOTION_DIRECTION_INFO), + STAGE_SPRITE: this._buildMenu(this.STAGE_SPRITE_INFO) + } + }; + } + + videoOn (args, util) { + this.detect.analyzeFrame(); + + let state = this.detect; + if (Number(args.STAGE_SPRITE) === 2) { + const drawable = this.runtime.renderer._allDrawables[util.target.drawableID]; + state = this._getMotionState(util.target); + this.detect.getLocalMotion(drawable, state); + } + + if (Number(args.MOTION_DIRECTION) === 1) { + return state.motionAmount; + } + return state.motionDirection; + } + + /** + * Check if the stack timer needs initialization. + * @param {object} util - utility object provided by the runtime. + * @return {boolean} - true if the stack timer needs to be initialized. + * @private + */ + _stackTimerNeedsInit (util) { + return !util.stackFrame.timer; + } + + /** + * Start the stack timer and the yield the thread if necessary. + * @param {object} util - utility object provided by the runtime. + * @param {number} duration - a duration in seconds to set the timer for. + * @private + */ + _startStackTimer (util, duration) { + util.stackFrame.timer = new Timer(); + util.stackFrame.timer.start(); + util.stackFrame.duration = duration; + util.yield(); + } + + /** + * Check the stack timer, and if its time is not up yet, yield the thread. + * @param {object} util - utility object provided by the runtime. + * @private + */ + _checkStackTimer (util) { + const timeElapsed = util.stackFrame.timer.timeElapsed(); + if (timeElapsed < util.stackFrame.duration * 1000) { + util.yield(); + } + } +} + +module.exports = Scratch3VideoSensingBlocks; diff --git a/src/extensions/scratch3_video_sensing/lib.js b/src/extensions/scratch3_video_sensing/lib.js new file mode 100644 index 000000000..611315e1d --- /dev/null +++ b/src/extensions/scratch3_video_sensing/lib.js @@ -0,0 +1,248 @@ +/** + * lib.js + * + * Tony Hwang and John Maloney, January 2011 + * Michael "Z" Goddard, March 2018 + * + * Video motion sensing primitives. + */ + +const TO_DEGREE = 180 / Math.PI; +const WIDTH = 480; +const HEIGHT = 360; +// chosen empirically to give a range of roughly 0-100 +const AMOUNT_SCALE = 100; +// note 2e-4 * activePixelNum is an experimentally tuned threshold for my +// logitech Pro 9000 webcam - TTH +const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4; +const THRESHOLD = 10; +const WINSIZE = 8; +const LOCAL_MAX_AMOUNT = 100; +const LOCAL_THRESHOLD = THRESHOLD / 3; + +const STATE_KEY = 'Scratch.videoSensing'; + +class VideoMotion { + constructor () { + this.frameNumber = 0; + this.motionAmount = 0; + this.motionDirection = 0; + this.analysisDone = false; + + this.curr = null; + this.prev = null; + + this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1); + this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT); + this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT); + } + + reset () { + this.prev = this.curr = null; + this.motionAmount = this.motionDirection = 0; + this.analysisDone = true; + + const targets = this.runtime.targets; + for (let i = 0; i < targets.length; i++) { + targets[i].getCustomState(STATE_KEY).motionAmount = 0; + targets[i].getCustomState(STATE_KEY).motionDirection = 0; + } + } + + addFrame (source) { + this.frameNumber++; + + this.prev = this.curr; + this.curr = new Uint32Array(source.buffer.slice()); + + const _tmp = this._prev; + this._prev = this._curr; + this._curr = _tmp; + for (let i = 0; i < this.curr.length; i++) { + this._curr[i] = this.curr[i] & 0xff; + } + + this.analysisDone = false; + } + + analyzeFrame () { + if (!this.curr || !this.prev) { + this.motionAmount = this.motionDirection = -1; + // don't have two frames to analyze yet + return; + } + + const { + _curr: curr, + _prev: prev + } = this; + + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + let uu = 0; + let vv = 0; + let n = 0; + + for (let i = WINSIZE + 1; i < hmax; i += winStep) { + for (let j = WINSIZE + 1; j < wmax; j += winStep) { + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C1 = 0; + let C2 = 0; + + let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE; + let nextAddress = address + winStep; + const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE; + for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) { + for (; address <= nextAddress; address += 1) { + const gradT = ((prev[address]) - (curr[address])); + const gradX = ((curr[address - 1]) - (curr[address + 1])); + const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); + + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + } + } + + const delta = ((A1B2 * A1B2) - (A2 * B1)); + let u = 0; + let v = 0; + if (delta) { + // system is not singular - solving by Kramer method + const deltaX = -((C1 * A1B2) - (C2 * B1)); + const deltaY = -((A1B2 * C2) - (A2 * C1)); + const Idelta = 8 / delta; + u = deltaX * Idelta; + v = deltaY * Idelta; + } else { + // singular system - find optical flow in gradient direction + const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); + if (Norm) { + const IGradNorm = 8 / Norm; + const temp = -(C1 + C2) * IGradNorm; + u = (A1B2 + A2) * temp; + v = (B1 + A1B2) * temp; + } + } + + if (-winStep < u && u < winStep && -winStep < v && v < winStep) { + uu += u; + vv += v; + n++; + } + } + } + + uu /= n; + vv /= n; + this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv)); + if (this.motionAmount > THRESHOLD) { + // Scratch direction + this.motionDirection = (((Math.atan2(vv, uu) * TO_DEGREE) + 270) % 360) - 180; + } + this.analysisDone = true; + } + + getLocalMotion (drawable, state) { + if (!this.curr || !this.prev) { + state.motionAmount = state.motionDirection = -1; + // don't have two frames to analyze yet + return; + } + if (state.motionFrameNumber !== this.frameNumber) { + const { + _prev: prev, + _curr: curr + } = this; + + const boundingRect = drawable.getFastBounds(); + const xmin = Math.floor(boundingRect.left + (WIDTH / 2)); + const xmax = Math.floor(boundingRect.right + (WIDTH / 2)); + const ymin = Math.floor((HEIGHT / 2) - boundingRect.top); + const ymax = Math.floor((HEIGHT / 2) - boundingRect.bottom); + + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C1 = 0; + let C2 = 0; + let scaleFactor = 0; + + const position = [0, 0, 0]; + + for (let i = ymin; i < ymax; i++) { + for (let j = xmin; j < xmax; j++) { + position[0] = j - (WIDTH / 2); + position[1] = (HEIGHT / 2) - i; + if ( + j > 0 && (j < WIDTH - 1) && + i > 0 && (i < HEIGHT - 1) && + drawable.isTouching(position) + ) { + const address = (i * WIDTH) + j; + const gradT = ((prev[address]) - (curr[address])); + const gradX = ((curr[address - 1]) - (curr[address + 1])); + const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH])); + + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + scaleFactor++; + } + } + } + + const delta = ((A1B2 * A1B2) - (A2 * B1)); + let u = 0; + let v = 0; + if (delta) { + // system is not singular - solving by Kramer method + const deltaX = -((C1 * A1B2) - (C2 * B1)); + const deltaY = -((A1B2 * C2) - (A2 * C1)); + const Idelta = 8 / delta; + u = deltaX * Idelta; + v = deltaY * Idelta; + } else { + // singular system - find optical flow in gradient direction + const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); + if (Norm) { + const IGradNorm = 8 / Norm; + const temp = -(C1 + C2) * IGradNorm; + u = (A1B2 + A2) * temp; + v = (B1 + A1B2) * temp; + } + } + + let activePixelNum = 0; + if (scaleFactor) { + // store the area of the sprite in pixels + activePixelNum = scaleFactor; + scaleFactor /= (2 * WINSIZE * 2 * WINSIZE); + + u = u / scaleFactor; + v = v / scaleFactor; + } + + state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v)); + if (state.motionAmount > LOCAL_MAX_AMOUNT) { + // clip all magnitudes greater than 100 + state.motionAmount = LOCAL_MAX_AMOUNT; + } + if (state.motionAmount > LOCAL_THRESHOLD) { + // Scratch direction + state.motionDirection = (((Math.atan2(v, u) * TO_DEGREE) + 270) % 360) - 180; + } + state.motionFrameNumber = this.frameNumber; + } + } +} + +module.exports = VideoMotion; diff --git a/src/extensions/scratch3_video_sensing/view.js b/src/extensions/scratch3_video_sensing/view.js new file mode 100644 index 000000000..665a9acfa --- /dev/null +++ b/src/extensions/scratch3_video_sensing/view.js @@ -0,0 +1,263 @@ +const WIDTH = 480; +const HEIGHT = 360; +const WINSIZE = 8; +const AMOUNT_SCALE = 100; +const THRESHOLD = 10; + +const OUTPUT = { + INPUT: -1, + XYT: 0, + XYT_CELL: 1, + XY: 2, + XY_CELL: 3, + T: 4, + T_CELL: 5, + C: 6, + AB: 7, + UV: 8 +}; + +class VideoMotionView { + constructor (motion, output = OUTPUT.XYT) { + this.motion = motion; + + const canvas = this.canvas = document.createElement('canvas'); + canvas.width = WIDTH; + canvas.height = HEIGHT; + this.context = canvas.getContext('2d'); + + this.output = output; + this.buffer = new Uint32Array(WIDTH * HEIGHT); + } + + static get OUTPUT () { + return OUTPUT; + } + + _eachAddress (xStart, yStart, xStop, yStop, fn) { + for (let i = yStart; i < yStop; i++) { + for (let j = xStart; j < xStop; j++) { + const address = (i * WIDTH) + j; + fn(address, j, i); + } + } + } + + _eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) { + const xStep2 = (xStep / 2) | 0; + const yStep2 = (yStep / 2) | 0; + for (let i = yStart; i < yStop; i += yStep) { + for (let j = xStart; j < xStop; j += xStep) { + fn( + _fn => this._eachAddress(j - xStep2 - 1, i - yStep2 - 1, j + xStep2, i + yStep2, _fn), + j - xStep2 - 1, + i - yStep2 - 1, + j + xStep2, + i + yStep2 + ); + } + } + } + + _grads (address) { + const {curr, prev} = this.motion; + const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff); + const gradY = (curr[address - WIDTH] & 0xff) - (curr[address + WIDTH] & 0xff); + const gradT = (prev[address] & 0xff) - (curr[address] & 0xff); + return {gradX, gradY, gradT}; + } + + draw () { + if (!(this.motion.prev && this.motion.curr)) { + return; + } + + const {buffer} = this; + + if (this.output === OUTPUT.INPUT) { + const {curr} = this.motion; + this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { + buffer[address] = curr[address]; + }); + } + if (this.output === OUTPUT.XYT) { + this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { + const {gradX, gradY, gradT} = this._grads(address); + const over1 = gradT / 0xcf; + buffer[address] = + (0xff << 24) + + (Math.floor((((gradY * over1) & 0xff) + 0xff) / 2) << 8) + + Math.floor((((gradX * over1) & 0xff) + 0xff) / 2); + }); + } + if (this.output === OUTPUT.XYT_CELL) { + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { + let C1 = 0; + let C2 = 0; + let n = 0; + + eachAddress(address => { + const {gradX, gradY, gradT} = this._grads(address); + C2 += (Math.max(Math.min(gradX / 0x0f, 1), -1)) * (gradT / 0xff); + C1 += (Math.max(Math.min(gradY / 0x0f, 1), -1)) * (gradT / 0xff); + n += 1; + }); + + C1 /= n; + C2 /= n; + C1 = Math.log(C1 + (1 * Math.sign(C1))) / Math.log(2); + C2 = Math.log(C2 + (1 * Math.sign(C2))) / Math.log(2); + + eachAddress(address => { + buffer[address] = (0xff << 24) + + (((((C1 * 0x7f) | 0) + 0x80) << 8) & 0xff00) + + (((((C2 * 0x7f) | 0) + 0x80) << 0) & 0xff); + }); + }); + } + if (this.output === OUTPUT.XY) { + this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { + const {gradX, gradY} = this._grads(address); + buffer[address] = (0xff << 24) + (((gradY + 0xff) / 2) << 8) + ((gradX + 0xff) / 2); + }); + } + if (this.output === OUTPUT.XY_CELL) { + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { + let C1 = 0; + let C2 = 0; + let n = 0; + + eachAddress(address => { + const {gradX, gradY} = this._grads(address); + C2 += Math.max(Math.min(gradX / 0x1f, 1), -1); + C1 += Math.max(Math.min(gradY / 0x1f, 1), -1); + n += 1; + }); + + C1 /= n; + C2 /= n; + C1 = Math.log(C1 + (1 * Math.sign(C1))) / Math.log(2); + C2 = Math.log(C2 + (1 * Math.sign(C2))) / Math.log(2); + + eachAddress(address => { + buffer[address] = (0xff << 24) + + (((((C1 * 0x7f) | 0) + 0x80) << 8) & 0xff00) + + (((((C2 * 0x7f) | 0) + 0x80) << 0) & 0xff); + }); + }); + } else if (this.output === OUTPUT.T) { + this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { + const {gradT} = this._grads(address); + buffer[address] = (0xff << 24) + ((gradT + 0xff) / 2 << 16); + }); + } + if (this.output === OUTPUT.T_CELL) { + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { + let T = 0; + let n = 0; + + eachAddress(address => { + const {gradT} = this._grads(address); + T += gradT / 0xff; + n += 1; + }); + + T /= n; + + eachAddress(address => { + buffer[address] = (0xff << 24) + + (((((T * 0x7f) | 0) + 0x80) << 16) & 0xff0000); + }); + }); + } else if (this.output === OUTPUT.C) { + this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { + const {gradX, gradY, gradT} = this._grads(address); + buffer[address] = + (0xff << 24) + + ((gradY * gradT) << 8) + + (gradX * gradT); + }); + } else if (this.output === OUTPUT.AB) { + this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => { + const {gradX, gradY} = this._grads(address); + buffer[address] = + (0xff << 24) + + ((gradX * gradY) << 16) + + ((gradY * gradY) << 8) + + (gradX * gradX); + }); + } else if (this.output === OUTPUT.UV) { + const winStep = (WINSIZE * 2) + 1; + const wmax = WIDTH - WINSIZE - 1; + const hmax = HEIGHT - WINSIZE - 1; + + this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => { + let A2 = 0; + let A1B2 = 0; + let B1 = 0; + let C2 = 0; + let C1 = 0; + + eachAddress(address => { + const {gradX, gradY, gradT} = this._grads(address); + A2 += gradX * gradX; + A1B2 += gradX * gradY; + B1 += gradY * gradY; + C2 += gradX * gradT; + C1 += gradY * gradT; + }); + + const delta = ((A1B2 * A1B2) - (A2 * B1)); + let u = 0; + let v = 0; + if (delta) { + /* system is not singular - solving by Kramer method */ + const deltaX = -((C1 * A1B2) - (C2 * B1)); + const deltaY = -((A1B2 * C2) - (A2 * C1)); + const Idelta = 8 / delta; + u = deltaX * Idelta; + v = deltaY * Idelta; + } else { + /* singular system - find optical flow in gradient direction */ + const Norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2)); + if (Norm) { + const IGradNorm = 8 / Norm; + const temp = -(C1 + C2) * IGradNorm; + u = (A1B2 + A2) * temp; + v = (B1 + A1B2) * temp; + } + } + + const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep); + const hypot = Math.hypot(u, v); + const amount = AMOUNT_SCALE * hypot; + eachAddress(address => { + buffer[address] = + (0xff << 24) + + (inRange && amount > THRESHOLD ? + (((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) + + (((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) : + 0x8080 + ); + }); + }); + } + + const data = new ImageData(new Uint8ClampedArray(this.buffer.buffer), WIDTH, HEIGHT); + this.context.putImageData(data, 0, 0); + } +} + +module.exports = VideoMotionView; From c206dc3241c962c9967725723b1b5ea8dc71b345 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Thu, 22 Mar 2018 11:52:23 -0400 Subject: [PATCH 2/3] Add motion playground page for debugging video sensing --- src/playground/motion.html | 18 ++++++ src/playground/motion.js | 118 +++++++++++++++++++++++++++++++++++++ webpack.config.js | 7 ++- 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 src/playground/motion.html create mode 100644 src/playground/motion.js diff --git a/src/playground/motion.html b/src/playground/motion.html new file mode 100644 index 000000000..c79deb0e9 --- /dev/null +++ b/src/playground/motion.html @@ -0,0 +1,18 @@ + + + + Video Motion Test Playground + + + + + + + + + + + + + + diff --git a/src/playground/motion.js b/src/playground/motion.js new file mode 100644 index 000000000..8d801d755 --- /dev/null +++ b/src/playground/motion.js @@ -0,0 +1,118 @@ +(function () { + const video = document.createElement('video'); + navigator.getUserMedia({ + audio: false, + video: { + width: {min: 480, ideal: 640}, + height: {min: 360, ideal: 480} + } + }, stream => { + video.autoplay = true; + video.src = window.URL.createObjectURL(stream); + // Get the track to hint to the browser the stream needs to be running + // even though we don't add the video tag to the DOM. + stream.getTracks(); + video.addEventListener('play', () => { + video.width = video.videoWidth; + video.height = video.videoHeight; + }); + }, err => { + /* eslint no-console:0 */ + console.log(err); + }); + + const VideoMotion = window.Scratch3MotionDetect.VideoMotion; + const VideoMotionView = window.Scratch3MotionDetect.VideoMotionView; + + // Create motion detector + const motion = new VideoMotion(); + + // Create debug views that will render different slices of how the detector + // uses the a frame of input. + const OUTPUT = VideoMotionView.OUTPUT; + const outputKeys = Object.keys(OUTPUT); + const outputValues = Object.values(OUTPUT); + const views = outputValues + .map(output => new VideoMotionView(motion, output)); + const view = views[0]; + + const defaultViews = [OUTPUT.INPUT, OUTPUT.XY_CELL, OUTPUT.T_CELL, OUTPUT.UV]; + + const activators = document.createElement('div'); + activators.style.userSelect = 'none'; + outputValues.forEach((output, index) => { + const checkboxLabel = document.createElement('label'); + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; + checkbox.checked = defaultViews.indexOf(output) !== -1; + const checkboxSpan = document.createElement('span'); + checkboxSpan.innerText = outputKeys[index]; + checkboxLabel.appendChild(checkbox); + checkboxLabel.appendChild(checkboxSpan); + + const _view = views[index]; + _view.canvas.style.display = checkbox.checked ? '' : 'none'; + _view.active = checkbox.checked; + checkbox.onchange = event => { + _view.canvas.style.display = checkbox.checked ? '' : 'none'; + _view.active = checkbox.checked; + event.preventDefault(); + return false; + }; + + activators.appendChild(checkboxLabel); + }); + document.body.appendChild(activators); + + // Add a text line to display milliseconds per frame, motion value, and + // motion direction + const textEl = document.createElement('div'); + document.body.appendChild(textEl); + let textTimer = Date.now(); + + // Add the motion debug views to the dom after the text line, so the text + // appears first. + views.forEach(_view => document.body.appendChild(_view.canvas)); + + // Create a temporary canvas the video will be drawn to so the video's + // bitmap data can be transformed into a TypeArray. + const tempCanvas = document.createElement('canvas'); + tempCanvas.width = view.canvas.width; + tempCanvas.height = view.canvas.height; + const ctx = tempCanvas.getContext('2d'); + + const loop = function () { + const timeoutId = setTimeout(loop, 33); + + try { + // Get the bitmap data for the video frame + ctx.scale(-1, 1); + ctx.drawImage( + video, + 0, 0, video.width || video.clientWidth, video.height || video.clientHeight, + -480, 0, tempCanvas.width, tempCanvas.height + ); + ctx.resetTransform(); + const data = ctx.getImageData(0, 0, tempCanvas.width, tempCanvas.height); + + const b = performance.now(); + motion.addFrame(data.data); + motion.analyzeFrame(); + if (Date.now() - textTimer > 250) { + const e = performance.now(); + const analyzeDuration = ((e - b) * 1000).toFixed(0); + const motionAmount = motion.motionAmount.toFixed(1); + const motionDirection = motion.motionDirection.toFixed(1); + textEl.innerText = `${analyzeDuration} :: ${motionAmount} :: ${motionDirection}`; + textTimer = Date.now(); + } + views.forEach(_view => _view.active && _view.draw()); + } catch (error) { + /* eslint no-console:0 */ + console.error(error.stack || error); + clearTimeout(timeoutId); + } + }; + + loop(); +}()); diff --git a/webpack.config.js b/webpack.config.js index 394b6e2a2..9580fc023 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -86,7 +86,8 @@ module.exports = [ 'scratch-storage', // Renderer 'scratch-render' - ] + ], + 'motion-extension': './src/extensions/scratch3_video_sensing/debug' }, output: { path: path.resolve(__dirname, 'playground'), @@ -98,6 +99,10 @@ module.exports = [ test: require.resolve('./src/index.js'), loader: 'expose-loader?VirtualMachine' }, + { + test: require.resolve('./src/extensions/scratch3_video_sensing/debug.js'), + loader: 'expose-loader?Scratch3MotionDetect' + }, { test: require.resolve('stats.js/build/stats.min.js'), loader: 'script-loader' From f3d19dc1a16e9e240df6e3fafb7cd37678fcef8b Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 26 Mar 2018 15:32:05 -0400 Subject: [PATCH 3/3] Use videoSensing extension for sb2 senseVideoMotion --- src/serialization/sb2.js | 16 ++++++++++++ src/serialization/sb2_specmap.js | 45 +++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/serialization/sb2.js b/src/serialization/sb2.js index cce5bb04b..2bc0ff3cb 100644 --- a/src/serialization/sb2.js +++ b/src/serialization/sb2.js @@ -562,6 +562,22 @@ const parseBlock = function (sb2block, addBroadcastMsg, getVariableId, extension if (shadowObscured) { fieldValue = 1; } + } else if (expectedArg.inputOp === 'videoSensing.menu.MOTION_DIRECTION') { + if (shadowObscured) { + fieldValue = 1; + } else if (fieldValue === 'motion') { + fieldValue = 1; + } else if (fieldValue === 'direction') { + fieldValue = 2; + } + } else if (expectedArg.inputOp === 'videoSensing.menu.STAGE_SPRITE') { + if (shadowObscured) { + fieldValue = 2; + } else if (fieldValue === 'Stage') { + fieldValue = 1; + } else if (fieldValue === 'this sprite') { + fieldValue = 2; + } } else if (shadowObscured) { // Filled drop-down menu. fieldValue = ''; diff --git a/src/serialization/sb2_specmap.js b/src/serialization/sb2_specmap.js index 637ccbab9..e302b68c4 100644 --- a/src/serialization/sb2_specmap.js +++ b/src/serialization/sb2_specmap.js @@ -609,6 +609,21 @@ const specMap = { } ] }, + 'senseVideoMotion': { + opcode: 'videoSensing.videoOn', + argMap: [ + { + type: 'input', + inputOp: 'videoSensing.menu.MOTION_DIRECTION', + inputName: 'MOTION_DIRECTION' + }, + { + type: 'input', + inputOp: 'videoSensing.menu.STAGE_SPRITE', + inputName: 'STAGE_SPRITE' + } + ] + }, 'whenGreenFlag': { opcode: 'event_whenflagclicked', argMap: [ @@ -905,21 +920,21 @@ const specMap = { argMap: [ ] }, - 'senseVideoMotion': { - opcode: 'sensing_videoon', - argMap: [ - { - type: 'input', - inputOp: 'sensing_videoonmenuone', - inputName: 'VIDEOONMENU1' - }, - { - type: 'input', - inputOp: 'sensing_videoonmenutwo', - inputName: 'VIDEOONMENU2' - } - ] - }, + // 'senseVideoMotion': { + // opcode: 'sensing_videoon', + // argMap: [ + // { + // type: 'input', + // inputOp: 'sensing_videoonmenuone', + // inputName: 'VIDEOONMENU1' + // }, + // { + // type: 'input', + // inputOp: 'sensing_videoonmenutwo', + // inputName: 'VIDEOONMENU2' + // } + // ] + // }, 'setVideoState': { opcode: 'sensing_videotoggle', argMap: [