diff --git a/src/engine/runtime.js b/src/engine/runtime.js index 677ff36a2..7c400e0e4 100644 --- a/src/engine/runtime.js +++ b/src/engine/runtime.js @@ -19,6 +19,7 @@ const DeviceManager = require('../io/deviceManager'); const Keyboard = require('../io/keyboard'); const Mouse = require('../io/mouse'); const MouseWheel = require('../io/mouseWheel'); +const Video = require('../io/video'); const defaultBlockPackages = { scratch3_control: require('../blocks/scratch3_control'), @@ -248,7 +249,8 @@ class Runtime extends EventEmitter { deviceManager: new DeviceManager(), keyboard: new Keyboard(this), mouse: new Mouse(this), - mouseWheel: new MouseWheel(this) + mouseWheel: new MouseWheel(this), + video: new Video(this) }; /** diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 9b70c498f..f09468476 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -3,10 +3,26 @@ const Runtime = require('../../engine/runtime'); const ArgumentType = require('../../extension-support/argument-type'); const BlockType = require('../../extension-support/block-type'); const Clone = require('../../util/clone'); -const log = require('../../util/log'); +const Video = require('../../io/video'); const VideoMotion = require('./library'); +/** + * States the video sensing activity can be set to. + * @readonly + * @enum {string} + */ +const VideoState = { + /** Video turned off. */ + OFF: 'off', + + /** Video turned on with default y axis mirroring. */ + ON: 'on', + + /** Video turned on without default y axis mirroring. */ + ON_FLIPPED: 'on-flipped' +}; + /** * Class for the motion-related blocks in Scratch 3.0 * @param {Runtime} runtime - the runtime instantiating this block package. @@ -34,39 +50,6 @@ class Scratch3VideoSensingBlocks { */ this._lastUpdate = null; - /** - * Id representing a Scratch Renderer skin the video is rendered to for - * previewing. - * @type {number} - */ - this._skinId = -1; - - /** - * The Scratch Renderer Skin object. - * @type {Skin} - */ - this._skin = null; - - /** - * Id for a drawable using the video's skin that will render as a video - * preview. - * @type {Drawable} - */ - this._drawable = -1; - - /** - * Canvas DOM element video is rendered to down or up sample to the - * expected resolution. - * @type {HTMLCanvasElement} - */ - this._sampleCanvas = null; - - /** - * Canvas 2D Context to render to the _sampleCanvas member. - * @type {CanvasRenderingContext2D} - */ - this._sampleContext = null; - if (this.runtime.ioDevices) { // Clear target motion state values when the project starts. this.runtime.on(Runtime.PROJECT_RUN_START, this.reset.bind(this)); @@ -74,9 +57,6 @@ class Scratch3VideoSensingBlocks { // Boot up the video, canvas to down/up sample the video stream, the // preview skin and drawable, and kick off looping the analysis // logic. - this._setupVideo(); - this._setupSampleCanvas(); - this._setupPreview(); this._loop(); } } @@ -99,14 +79,6 @@ class Scratch3VideoSensingBlocks { return [480, 360]; } - /** - * Order preview drawable is inserted at in the renderer. - * @type {number} - */ - static get ORDER () { - return 1; - } - /** * The key to load & store a target's motion-related state. * @type {string} @@ -145,127 +117,29 @@ class Scratch3VideoSensingBlocks { } } - /** - * Setup a video element connected to a user media stream. - * @private - */ - _setupVideo () { - this._video = document.createElement('video'); - navigator.getUserMedia({ - audio: false, - video: { - width: {min: 480, ideal: 640}, - height: {min: 360, ideal: 480} - } - }, stream => { - this._video.src = window.URL.createObjectURL(stream); - // Hint to the stream that it should load. A standard way to do this - // is add the video tag to the DOM. Since this extension wants to - // hide the video tag and instead render a sample of the stream into - // the webgl rendered Scratch canvas, another hint like this one is - // needed. - this._track = stream.getTracks()[0]; - }, err => { - // @todo Properly handle errors - log(err); - }); - } - - /** - * Create a campus to render the user media video to down/up sample to the - * needed resolution. - * @private - */ - _setupSampleCanvas () { - // Create low-resolution image to sample video for analysis and preview - const canvas = this._sampleCanvas = document.createElement('canvas'); - canvas.width = Scratch3VideoSensingBlocks.DIMENSIONS[0]; - canvas.height = Scratch3VideoSensingBlocks.DIMENSIONS[1]; - this._sampleContext = canvas.getContext('2d'); - } - - /** - * Create a Scratch Renderer Skin and Drawable to preview the user media - * video stream. - * @private - */ - _setupPreview () { - if (this._skinId !== -1) return; - if (this._skin !== null) return; - if (this._drawable !== -1) return; - if (!this.runtime.renderer) return; - - this._skinId = this.runtime.renderer.createPenSkin(); - this._skin = this.runtime.renderer._allSkins[this._skinId]; - this._drawable = this.runtime.renderer.createDrawable(); - this.runtime.renderer.setDrawableOrder( - this._drawable, - Scratch3VideoSensingBlocks.ORDER - ); - this.runtime.renderer.updateDrawableProperties(this._drawable, { - skinId: this._skinId - }); - } - /** * Occasionally step a loop to sample the video, stamp it to the preview * skin, and add a TypedArray copy of the canvas's pixel data. * @private */ _loop () { - setTimeout(this._loop.bind(this), this.runtime.currentStepTime); - - // Ensure video stream is established - if (!this._video) return; - if (!this._track) return; - if (typeof this._video.videoWidth !== 'number') return; - if (typeof this._video.videoHeight !== 'number') return; - - // Bail if the camera is *still* not ready - const nativeWidth = this._video.videoWidth; - const nativeHeight = this._video.videoHeight; - if (nativeWidth === 0) return; - if (nativeHeight === 0) return; - - const ctx = this._sampleContext; - - // Mirror - ctx.scale(-1, 1); - - // Generate video thumbnail for analysis - ctx.drawImage( - this._video, - 0, - 0, - nativeWidth, - nativeHeight, - Scratch3VideoSensingBlocks.DIMENSIONS[0] * -1, - 0, - Scratch3VideoSensingBlocks.DIMENSIONS[0], - Scratch3VideoSensingBlocks.DIMENSIONS[1] - ); - - // Restore the canvas transform - ctx.resetTransform(); - - // Render to preview layer - if (this._skin !== null) { - const xOffset = Scratch3VideoSensingBlocks.DIMENSIONS[0] / 2 * -1; - const yOffset = Scratch3VideoSensingBlocks.DIMENSIONS[1] / 2; - this._skin.drawStamp(this._sampleCanvas, xOffset, yOffset); - this.runtime.requestRedraw(); - } + setTimeout(this._loop.bind(this), Math.max(this.runtime.currentStepTime, Scratch3VideoSensingBlocks.INTERVAL)); // Add frame to detector const time = Date.now(); - if (this._lastUpdate === null) this._lastUpdate = time; + if (this._lastUpdate === null) { + this._lastUpdate = time; + } const offset = time - this._lastUpdate; if (offset > Scratch3VideoSensingBlocks.INTERVAL) { - this._lastUpdate = time; - const data = ctx.getImageData( - 0, 0, Scratch3VideoSensingBlocks.DIMENSIONS[0], Scratch3VideoSensingBlocks.DIMENSIONS[1] - ); - this.detect.addFrame(data.data); + const frame = this.runtime.ioDevices.video.getFrame({ + format: Video.FORMAT_IMAGE_DATA, + dimensions: Scratch3VideoSensingBlocks.DIMENSIONS + }); + if (frame) { + this._lastUpdate = time; + this.detect.addFrame(frame.data); + } } } @@ -282,7 +156,7 @@ class Scratch3VideoSensingBlocks { return info.map((entry, index) => { const obj = {}; obj.text = entry.name; - obj.value = String(index + 1); + obj.value = entry.value || String(index + 1); return obj; }); } @@ -355,6 +229,38 @@ class Scratch3VideoSensingBlocks { return 2; } + /** + * States the video sensing activity can be set to. + * @readonly + * @enum {string} + */ + static get VideoState () { + return VideoState; + } + + /** + * An array of info on video state options for the "turn video [STATE]" block. + * @type {object[]} an array of objects + * @param {string} name - the translatable name to display in the video state menu + * @param {string} value - the serializable value stored in the block + */ + get VIDEO_STATE_INFO () { + return [ + { + name: 'off', + value: VideoState.OFF + }, + { + name: 'on', + value: VideoState.ON + }, + { + name: 'on flipped', + value: VideoState.ON_FLIPPED + } + ]; + } + /** * @returns {object} metadata for this extension and its blocks. */ @@ -392,11 +298,33 @@ class Scratch3VideoSensingBlocks { defaultValue: 10 } } + }, + { + opcode: 'videoToggle', + text: 'turn video [VIDEO_STATE]', + arguments: { + VIDEO_STATE: { + type: ArgumentType.NUMBER, + menu: 'VIDEO_STATE', + defaultValue: VideoState.ON + } + } + }, + { + opcode: 'setVideoTransparency', + text: 'set video transparency to [TRANSPARENCY]', + arguments: { + TRANSPARENCY: { + type: ArgumentType.NUMBER, + defaultValue: 0 + } + } } ], menus: { MOTION_DIRECTION: this._buildMenu(this.MOTION_DIRECTION_INFO), - STAGE_SPRITE: this._buildMenu(this.STAGE_SPRITE_INFO) + STAGE_SPRITE: this._buildMenu(this.STAGE_SPRITE_INFO), + VIDEO_STATE: this._buildMenu(this.VIDEO_STATE_INFO) } }; } @@ -449,6 +377,34 @@ class Scratch3VideoSensingBlocks { const state = this._analyzeLocalMotion(util.target); return state.motionAmount > Number(args.REFERENCE); } + + /** + * A scratch command block handle that configures the video state from + * passed arguments. + * @param {object} args - the block arguments + * @param {VideoState} args.VIDEO_STATE - the video state to set the device to + */ + videoToggle (args) { + const state = args.VIDEO_STATE; + if (state === VideoState.OFF) { + this.runtime.ioDevices.video.disableVideo(); + } else { + this.runtime.ioDevices.video.enableVideo(); + // Mirror if state is ON. Do not mirror if state is ON_FLIPPED. + this.runtime.ioDevices.video.mirror = state === VideoState.ON; + } + } + + /** + * A scratch command block handle that configures the video preview's + * transparency from passed arguments. + * @param {object} args - the block arguments + * @param {number} args.TRANSPARENCY - the transparency to set the video + * preview to + */ + setVideoTransparency (args) { + this.runtime.ioDevices.video.setPreviewGhost(Number(args.TRANSPARENCY)); + } } module.exports = Scratch3VideoSensingBlocks; diff --git a/src/io/video.js b/src/io/video.js new file mode 100644 index 000000000..6c72be885 --- /dev/null +++ b/src/io/video.js @@ -0,0 +1,380 @@ +const log = require('../util/log'); + +class Video { + constructor (runtime) { + /** + * Reference to the owning Runtime. + * @type{!Runtime} + */ + this.runtime = runtime; + + /** + * Default value for mirrored frames. + * @type boolean + */ + this.mirror = true; + + /** + * Cache frames for this many ms. + * @type number + */ + this._frameCacheTimeout = 16; + + /** + * DOM Video element + * @private + */ + this._video = null; + + /** + * Usermedia stream track + * @private + */ + this._track = null; + + /** + * Stores some canvas/frame data per resolution/mirror states + */ + this._workspace = []; + + /** + * Id representing a Scratch Renderer skin the video is rendered to for + * previewing. + * @type {number} + */ + this._skinId = -1; + + /** + * The Scratch Renderer Skin object. + * @type {Skin} + */ + this._skin = null; + + /** + * Id for a drawable using the video's skin that will render as a video + * preview. + * @type {Drawable} + */ + this._drawable = -1; + + /** + * Store the last state of the video transparency ghost effect + * @type {number} + */ + this._ghost = 0; + } + + static get FORMAT_IMAGE_DATA () { + return 'image-data'; + } + + static get FORMAT_CANVAS () { + return 'canvas'; + } + + /** + * Dimensions the video stream is analyzed at after its rendered to the + * sample canvas. + * @type {Array.} + */ + static get DIMENSIONS () { + return [480, 360]; + } + + /** + * Order preview drawable is inserted at in the renderer. + * @type {number} + */ + static get ORDER () { + return 1; + } + + /** + * Request video be enabled. Sets up video, creates video skin and enables preview. + * + * ioDevices.video.requestVideo() + * + * @return {Promise.