From b46c956203cbb501359c2ef65ba5b2c720d539a3 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 30 Mar 2018 14:50:53 -0400 Subject: [PATCH] Comment items in Scratch3VideoSensingBlocks --- .../scratch3_video_sensing/index.js | 154 ++++++++++++++---- 1 file changed, 125 insertions(+), 29 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 2904f5f54..10658f0c6 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -1,3 +1,5 @@ +const Runtime = require('../../engine/runtime'); + const ArgumentType = require('../../extension-support/argument-type'); const BlockType = require('../../extension-support/block-type'); const Clone = require('../../util/clone'); @@ -32,32 +34,123 @@ class Scratch3VideoSensingBlocks { */ this.runtime = runtime; + /** + * The motion detection algoritm used to power the motion amount and + * direction values. + * @type {VideoMotion} + */ this.detect = new VideoMotion(); + /** + * The last millisecond epoch timestamp that the video stream was + * analyzed. + * @type {number} + */ this._lastUpdate = null; + /** + * Id representing a Scratch Renderer skin the video is rendered to for + * previewing. + * @type {number} + */ this._skinId = -1; + + /** + * The Scratch Renderer Skin object. + * @type {Skin} + */ this._skin = null; + + /** + * Id for a drawable using the video's skin that will render as a video + * preview. + * @type {Drawable} + */ this._drawable = -1; + /** + * Canvas DOM element video is rendered to down or up sample to the + * expected resolution. + * @type {HTMLCanvasElement} + */ + this._sampleCanvas = null; + + /** + * Canvas 2D Context to render to the _sampleCanvas member. + * @type {CanvasRenderingContext2D} + */ + this._sampleContext = null; + + // Clear target motion state values when the project starts. + this.runtime.on(Runtime.PROJECT_RUN_START, this.reset.bind(this)); + + // Boot up the video, canvas to down/up sample the video stream, the + // preview skin and drawable, and kick off looping the analysis logic. this._setupVideo(); this._setupSampleCanvas(); this._setupPreview(); this._loop(); } - static get INTERVAL () { - return 33; - } - + /** + * Dimensions the video stream is analyzed at after its rendered to the + * sample canvas. + * @type {Array.} + */ static get DIMENSIONS () { return [480, 360]; } + /** + * Order preview drawable is inserted at in the renderer. + * @type {number} + */ static get ORDER () { return 1; } + /** + * The key to load & store a target's motion-related state. + * @type {string} + */ + static get STATE_KEY () { + return 'Scratch.videoSensing'; + } + + /** + * The default motion-related state, to be used when a target has no existing motion state. + * @type {MotionState} + */ + static get DEFAULT_MOTION_STATE () { + return { + motionFrameNumber: 0, + motionAmount: 0, + motionDirection: 0 + }; + } + + /** + * Reset the extension's data motion detection data. This will clear out + * for example old frames, so the first analyzed frame will not be compared + * against a frame from before reset was called. + */ + reset () { + this.detect.reset(); + + const targets = this.runtime.targets; + for (let i = 0; i < targets.length; i++) { + const state = targets[i].getCustomState(Scratch3VideoSensingBlocks.STATE_KEY); + if (state) { + state.motionAmount = 0; + state.motionDirection = 0; + } + } + } + + /** + * Setup a video element connected to a user media stream. + * @private + */ _setupVideo () { this._video = document.createElement('video'); navigator.getUserMedia({ @@ -80,6 +173,11 @@ class Scratch3VideoSensingBlocks { }); } + /** + * Create a campus to render the user media video to down/up sample to the + * needed resolution. + * @private + */ _setupSampleCanvas () { // Create low-resolution image to sample video for analysis and preview const canvas = this._sampleCanvas = document.createElement('canvas'); @@ -88,6 +186,11 @@ class Scratch3VideoSensingBlocks { this._sampleContext = canvas.getContext('2d'); } + /** + * Create a Scratch Renderer Skin and Drawable to preview the user media + * video stream. + * @private + */ _setupPreview () { if (this._skinId !== -1) return; if (this._skin !== null) return; @@ -106,6 +209,11 @@ class Scratch3VideoSensingBlocks { }); } + /** + * Occasionally step a loop to sample the video, stamp it to the preview + * skin, and add a TypedArray copy of the canvas's pixel data. + * @private + */ _loop () { setTimeout(this._loop.bind(this), this.runtime.currentStepTime); @@ -164,9 +272,11 @@ class Scratch3VideoSensingBlocks { } /** - * Create data for a menu in scratch-blocks format, consisting of an array of objects with text and - * value properties. The text is a translated string, and the value is one-indexed. - * @param {object[]} info - An array of info objects each having a name property. + * Create data for a menu in scratch-blocks format, consisting of an array + * of objects with text and value properties. The text is a translated + * string, and the value is one-indexed. + * @param {object[]} info - An array of info objects each having a name + * property. * @return {array} - An array of objects with text and value properties. * @private */ @@ -179,27 +289,10 @@ class Scratch3VideoSensingBlocks { }); } - /** - * The key to load & store a target's motion-related state. - * @type {string} - */ - static get STATE_KEY () { - return 'Scratch.videoSensing'; - } - - /** - * The default music-related state, to be used when a target has no existing music state. - * @type {MusicState} - */ - static get DEFAULT_MOTION_STATE () { - return { - currentInstrument: 0 - }; - } - /** * @param {Target} target - collect motion state for this target. - * @returns {MotionState} the mutable motion state associated with that target. This will be created if necessary. + * @returns {MotionState} the mutable motion state associated with that + * target. This will be created if necessary. * @private */ _getMotionState (target) { @@ -212,7 +305,8 @@ class Scratch3VideoSensingBlocks { } /** - * An array of info about each drum. + * An array of choices of whether a reporter should return the frame's + * motion amount or direction. * @type {object[]} an array of objects. * @param {string} name - the translatable name to display in the drums menu. * @param {string} fileName - the name of the audio file containing the drum sound. @@ -231,8 +325,10 @@ class Scratch3VideoSensingBlocks { /** * An array of info about each drum. * @type {object[]} an array of objects. - * @param {string} name - the translatable name to display in the drums menu. - * @param {string} fileName - the name of the audio file containing the drum sound. + * @param {string} name - the translatable name to display in the drums + * menu. + * @param {string} fileName - the name of the audio file containing the + * drum sound. */ get STAGE_SPRITE_INFO () { return [