From eef88f6c2d5726bed04c114e37e86c0fce114207 Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 11:44:44 -0400 Subject: [PATCH 01/22] First draft of video IO device --- src/engine/runtime.js | 4 +- .../scratch3_video_sensing/index.js | 175 +-------- src/io/video.js | 332 ++++++++++++++++++ 3 files changed, 354 insertions(+), 157 deletions(-) create mode 100644 src/io/video.js diff --git a/src/engine/runtime.js b/src/engine/runtime.js index aa73c9690..7552c2d7e 100644 --- a/src/engine/runtime.js +++ b/src/engine/runtime.js @@ -15,6 +15,7 @@ const DeviceManager = require('../io/deviceManager'); const Keyboard = require('../io/keyboard'); const Mouse = require('../io/mouse'); const MouseWheel = require('../io/mouseWheel'); +const Video = require('../io/video'); const defaultBlockPackages = { scratch3_control: require('../blocks/scratch3_control'), @@ -260,7 +261,8 @@ class Runtime extends EventEmitter { deviceManager: new DeviceManager(), keyboard: new Keyboard(this), mouse: new Mouse(this), - mouseWheel: new MouseWheel(this) + mouseWheel: new MouseWheel(this), + video: new Video(this) }; /** diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 9b70c498f..3a276768a 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -3,7 +3,7 @@ const Runtime = require('../../engine/runtime'); const ArgumentType = require('../../extension-support/argument-type'); const BlockType = require('../../extension-support/block-type'); const Clone = require('../../util/clone'); -const log = require('../../util/log'); +const Video = require('../../io/video'); const VideoMotion = require('./library'); @@ -34,39 +34,6 @@ class Scratch3VideoSensingBlocks { */ this._lastUpdate = null; - /** - * Id representing a Scratch Renderer skin the video is rendered to for - * previewing. - * @type {number} - */ - this._skinId = -1; - - /** - * The Scratch Renderer Skin object. - * @type {Skin} - */ - this._skin = null; - - /** - * Id for a drawable using the video's skin that will render as a video - * preview. - * @type {Drawable} - */ - this._drawable = -1; - - /** - * Canvas DOM element video is rendered to down or up sample to the - * expected resolution. - * @type {HTMLCanvasElement} - */ - this._sampleCanvas = null; - - /** - * Canvas 2D Context to render to the _sampleCanvas member. - * @type {CanvasRenderingContext2D} - */ - this._sampleContext = null; - if (this.runtime.ioDevices) { // Clear target motion state values when the project starts. this.runtime.on(Runtime.PROJECT_RUN_START, this.reset.bind(this)); @@ -74,10 +41,11 @@ class Scratch3VideoSensingBlocks { // Boot up the video, canvas to down/up sample the video stream, the // preview skin and drawable, and kick off looping the analysis // logic. - this._setupVideo(); - this._setupSampleCanvas(); - this._setupPreview(); - this._loop(); + this.runtime.ioDevices.video.requestVideo() + .then(({release}) => { + this.releaseVideo = release; + this._loop(); + }); } } @@ -99,14 +67,6 @@ class Scratch3VideoSensingBlocks { return [480, 360]; } - /** - * Order preview drawable is inserted at in the renderer. - * @type {number} - */ - static get ORDER () { - return 1; - } - /** * The key to load & store a target's motion-related state. * @type {string} @@ -145,127 +105,30 @@ class Scratch3VideoSensingBlocks { } } - /** - * Setup a video element connected to a user media stream. - * @private - */ - _setupVideo () { - this._video = document.createElement('video'); - navigator.getUserMedia({ - audio: false, - video: { - width: {min: 480, ideal: 640}, - height: {min: 360, ideal: 480} - } - }, stream => { - this._video.src = window.URL.createObjectURL(stream); - // Hint to the stream that it should load. A standard way to do this - // is add the video tag to the DOM. Since this extension wants to - // hide the video tag and instead render a sample of the stream into - // the webgl rendered Scratch canvas, another hint like this one is - // needed. - this._track = stream.getTracks()[0]; - }, err => { - // @todo Properly handle errors - log(err); - }); - } - - /** - * Create a campus to render the user media video to down/up sample to the - * needed resolution. - * @private - */ - _setupSampleCanvas () { - // Create low-resolution image to sample video for analysis and preview - const canvas = this._sampleCanvas = document.createElement('canvas'); - canvas.width = Scratch3VideoSensingBlocks.DIMENSIONS[0]; - canvas.height = Scratch3VideoSensingBlocks.DIMENSIONS[1]; - this._sampleContext = canvas.getContext('2d'); - } - - /** - * Create a Scratch Renderer Skin and Drawable to preview the user media - * video stream. - * @private - */ - _setupPreview () { - if (this._skinId !== -1) return; - if (this._skin !== null) return; - if (this._drawable !== -1) return; - if (!this.runtime.renderer) return; - - this._skinId = this.runtime.renderer.createPenSkin(); - this._skin = this.runtime.renderer._allSkins[this._skinId]; - this._drawable = this.runtime.renderer.createDrawable(); - this.runtime.renderer.setDrawableOrder( - this._drawable, - Scratch3VideoSensingBlocks.ORDER - ); - this.runtime.renderer.updateDrawableProperties(this._drawable, { - skinId: this._skinId - }); - } - /** * Occasionally step a loop to sample the video, stamp it to the preview * skin, and add a TypedArray copy of the canvas's pixel data. * @private */ _loop () { - setTimeout(this._loop.bind(this), this.runtime.currentStepTime); - - // Ensure video stream is established - if (!this._video) return; - if (!this._track) return; - if (typeof this._video.videoWidth !== 'number') return; - if (typeof this._video.videoHeight !== 'number') return; - - // Bail if the camera is *still* not ready - const nativeWidth = this._video.videoWidth; - const nativeHeight = this._video.videoHeight; - if (nativeWidth === 0) return; - if (nativeHeight === 0) return; - - const ctx = this._sampleContext; - - // Mirror - ctx.scale(-1, 1); - - // Generate video thumbnail for analysis - ctx.drawImage( - this._video, - 0, - 0, - nativeWidth, - nativeHeight, - Scratch3VideoSensingBlocks.DIMENSIONS[0] * -1, - 0, - Scratch3VideoSensingBlocks.DIMENSIONS[0], - Scratch3VideoSensingBlocks.DIMENSIONS[1] - ); - - // Restore the canvas transform - ctx.resetTransform(); - - // Render to preview layer - if (this._skin !== null) { - const xOffset = Scratch3VideoSensingBlocks.DIMENSIONS[0] / 2 * -1; - const yOffset = Scratch3VideoSensingBlocks.DIMENSIONS[1] / 2; - this._skin.drawStamp(this._sampleCanvas, xOffset, yOffset); - this.runtime.requestRedraw(); - } + setTimeout(this._loop.bind(this), Math.max(this.runtime.currentStepTime, Scratch3VideoSensingBlocks.INTERVAL)); // Add frame to detector const time = Date.now(); - if (this._lastUpdate === null) this._lastUpdate = time; + if (this._lastUpdate === null) { + this._lastUpdate = time; + } const offset = time - this._lastUpdate; if (offset > Scratch3VideoSensingBlocks.INTERVAL) { - this._lastUpdate = time; - const data = ctx.getImageData( - 0, 0, Scratch3VideoSensingBlocks.DIMENSIONS[0], Scratch3VideoSensingBlocks.DIMENSIONS[1] - ); - this.detect.addFrame(data.data); + const frame = this.runtime.ioDevices.video.getFrame({ + mirror: true, + format: Video.FORMAT_IMAGE_DATA, + dimensions: Scratch3VideoSensingBlocks.DIMENSIONS + }); + if (frame) { + this._lastUpdate = time; + this.detect.addFrame(frame.data); + } } } diff --git a/src/io/video.js b/src/io/video.js new file mode 100644 index 000000000..59b04d6f0 --- /dev/null +++ b/src/io/video.js @@ -0,0 +1,332 @@ +const log = require('../util/log'); + +class Video { + constructor (runtime) { + /** + * Reference to the owning Runtime. + * @type{!Runtime} + */ + this.runtime = runtime; + + /** + * Cache frames for this many ms. + * @type number + */ + this._frameCacheTimeout = 16; + + /** + * Store each request for video, so when all are released we can disable preview/video feed. + * @type Array. + */ + this._requests = []; + + /** + * DOM Video element + * @private + */ + this._video = null; + + /** + * Usermedia stream track + * @private + */ + this._track = null; + + /** + * Stores some canvas/frame data per resolution/mirror states + */ + this._workspace = []; + + /** + * Id representing a Scratch Renderer skin the video is rendered to for + * previewing. + * @type {number} + */ + this._skinId = -1; + + /** + * The Scratch Renderer Skin object. + * @type {Skin} + */ + this._skin = null; + + /** + * Id for a drawable using the video's skin that will render as a video + * preview. + * @type {Drawable} + */ + this._drawable = -1; + } + + static get FORMAT_IMAGE_DATA () { + return 'image-data'; + } + + static get FORMAT_CANVAS () { + return 'canvas'; + } + + /** + * Dimensions the video stream is analyzed at after its rendered to the + * sample canvas. + * @type {Array.} + */ + static get DIMENSIONS () { + return [480, 360]; + } + + /** + * Order preview drawable is inserted at in the renderer. + * @type {number} + */ + static get ORDER () { + return 1; + } + + /** + * Request video be enabled. Sets up video, creates video skin and enables preview. + * + * ioDevices.video.requestVideo() + * .then(({ release }) => { + * this.releaseVideo = release; + * }) + * + * @return {Promise.} A request object with a "release" property that + * should be called when you are done with the video. + */ + requestVideo () { + const io = this; + const request = { + release () { + const index = io._requests.indexOf(request); + if (index > -1) { + io._requests.splice(index, 1); + } + if (io._requests.length === 0) { + io._disablePreview(); + // by clearing refs to video and track, we should lose our hold over the camera + io._video = null; + io._track = null; + } + } + }; + + if (this.videoReady) { + this._requests.push(request); + return Promise.resolve(request); + } + + if (this._lastSetup) { + return this._lastSetup.then(() => { + this._requests.push(request); + return request; + }); + } + + this._lastSetup = this._setupVideo() + .then(() => { + this._setupPreview(); + this._requests.push(request); + this._lastSetup = null; + return request; + }, err => { + this._lastSetup = null; + throw err; + }); + return this._lastSetup; + } + + /** + * Create a video stream. + * Should probably be moved to -render or somewhere similar later + * @private + * @return {Promise} When video has been received, rejected if video is not received + */ + _setupVideo () { + this._video = document.createElement('video'); + return new Promise((resolve, reject) => { + navigator.getUserMedia({ + audio: false, + video: { + width: {min: 480, ideal: 640}, + height: {min: 360, ideal: 480} + } + }, stream => { + this._video.src = window.URL.createObjectURL(stream); + // Hint to the stream that it should load. A standard way to do this + // is add the video tag to the DOM. Since this extension wants to + // hide the video tag and instead render a sample of the stream into + // the webgl rendered Scratch canvas, another hint like this one is + // needed. + this._track = stream.getTracks()[0]; + resolve(this._video); + }, err => { + // There are probably some error types we could handle gracefully here. + reject(err); + }); + }); + } + + _disablePreview () { + if (this._skin) { + this._skin.clear(); + } + this._renderPreviewFrame = null; + } + + _setupPreview () { + const {renderer} = this.runtime; + if (!renderer) return; + + if (this._skinId === -1 && this._skin === null && this._drawable === -1) { + this._skinId = renderer.createPenSkin(); + this._skin = renderer._allSkins[this._skinId]; + this._drawable = renderer.createDrawable(); + renderer.setDrawableOrder( + this._drawable, + Video.ORDER + ); + renderer.updateDrawableProperties(this._drawable, { + skinId: this._skinId + }); + } + + // if we haven't already created and started a preview frame render loop, do so + if (!this._renderPreviewFrame) { + this._renderPreviewFrame = () => { + if (!this._renderPreviewFrame) { + return; + } + + setTimeout(this._renderPreviewFrame, this.runtime.currentStepTime); + + const canvas = this.getFrame({format: Video.FORMAT_CANVAS}); + + if (!canvas) { + return; + } + + const xOffset = Video.DIMENSIONS[0] / -2; + const yOffset = Video.DIMENSIONS[1] / 2; + this._skin.drawStamp(canvas, xOffset, yOffset); + this.runtime.requestRedraw(); + }; + + this._renderPreviewFrame(); + } + + } + + get videoReady () { + if (!this._video) { + return false; + } + if (!this._track) { + return false; + } + const {videoWidth, videoHeight} = this._video; + if (typeof videoWidth !== 'number' || typeof videoHeight !== 'number') { + return false; + } + if (videoWidth === 0 || videoHeight === 0) { + return false; + } + return true; + } + + /** + * get an internal workspace for canvas/context/caches + * this uses some document stuff to create a canvas and what not, probably needs abstraction + * into the renderer layer? + * @private + * @return {object} A workspace for canvas/data storage. Internal format not documented intentionally + */ + _getWorkspace ({dimensions, mirror}) { + let workspace = this._workspace.find(space => ( + space.dimensions.join('-') === dimensions.join('-') && + space.mirror === mirror + )); + if (!workspace) { + workspace = { + dimensions, + mirror, + canvas: document.createElement('canvas'), + lastUpdate: 0, + cacheData: {} + }; + workspace.canvas.width = dimensions[0]; + workspace.canvas.height = dimensions[1]; + workspace.context = workspace.canvas.getContext('2d'); + this._workspace.push(workspace); + } + return workspace; + } + + /** + * Return frame data from the video feed in a specified dimensions, format, and mirroring. + * @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors. + */ + getFrame ({ + dimensions = Video.DIMENSIONS, + mirror = true, + format = Video.FORMAT_IMAGE_DATA, + cacheTimeout = this._frameCacheTimeout + }) { + if (!this.videoReady) { + return null; + } + const [width, height] = dimensions; + const workspace = this._getWorkspace({dimensions, mirror: Boolean(mirror)}); + const {videoWidth, videoHeight} = this._video; + const {canvas, context, lastUpdate, cacheData} = workspace; + const now = Date.now(); + + // if the canvas hasn't been updated... + if (lastUpdate + cacheTimeout < now) { + + if (mirror) { + context.scale(-1, 1); + context.translate(width * -1, 0); + } + + context.drawImage(this._video, + // source x, y, width, height + 0, 0, videoWidth, videoHeight, + // dest x, y, width, height + 0, 0, width, height + ); + + context.resetTransform(); + workspace.lastUpdate = now; + } + + // each data type has it's own data cache, but the canvas is the same + if (!cacheData[format]) { + cacheData[format] = {lastUpdate: 0}; + } + const formatCache = cacheData[format]; + + if (formatCache.lastUpdate + cacheTimeout < now) { + if (format === Video.FORMAT_IMAGE_DATA) { + formatCache.lastData = context.getImageData(0, 0, width, height); + } else if (format === Video.FORMAT_CANVAS) { + // this will never change + formatCache.lastUpdate = Infinity; + formatCache.lastData = canvas; + } else { + log.error(`video io error - unimplemented format ${format}`); + // cache the null result forever, don't log about it again.. + formatCache.lastUpdate = Infinity; + formatCache.lastData = null; + } + + // rather than set to now, this data is as stale as it's canvas is + formatCache.lastUpdate = Math.max(workspace.lastUpdate, formatCache.lastUpdate); + } + + return formatCache.lastData; + } +} + + +module.exports = Video; From c10696f88c9c0a1377867e5a8190155352650240 Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 15:07:50 -0400 Subject: [PATCH 02/22] Rewrite the setup/disable process from review comments --- src/io/video.js | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/io/video.js b/src/io/video.js index 59b04d6f0..f8957dd62 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -103,10 +103,7 @@ class Video { io._requests.splice(index, 1); } if (io._requests.length === 0) { - io._disablePreview(); - // by clearing refs to video and track, we should lose our hold over the camera - io._video = null; - io._track = null; + io._disableVideo(); } } }; @@ -116,24 +113,10 @@ class Video { return Promise.resolve(request); } - if (this._lastSetup) { - return this._lastSetup.then(() => { - this._requests.push(request); - return request; - }); - } - - this._lastSetup = this._setupVideo() - .then(() => { - this._setupPreview(); - this._requests.push(request); - this._lastSetup = null; - return request; - }, err => { - this._lastSetup = null; - throw err; - }); - return this._lastSetup; + return this._setupVideo().then(() => { + this._requests.push(request); + return request; + }); } /** @@ -143,8 +126,12 @@ class Video { * @return {Promise} When video has been received, rejected if video is not received */ _setupVideo () { + if (this._lastSetup) { + return this._lastSetup; + } + this._video = document.createElement('video'); - return new Promise((resolve, reject) => { + const video = new Promise((resolve, reject) => { navigator.getUserMedia({ audio: false, video: { @@ -162,9 +149,20 @@ class Video { resolve(this._video); }, err => { // There are probably some error types we could handle gracefully here. + this._lastSetup = null; reject(err); }); }); + + return video.then(() => this._setupPreview()); + } + + _disableVideo () { + this._disablePreview(); + this._lastSetup = null; + // by clearing refs to video and track, we should lose our hold over the camera + this._video = null; + this._track = null; } _disablePreview () { From da05e673fac1b52b08c11cd5e0ad13475ebfd396 Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 15:09:25 -0400 Subject: [PATCH 03/22] _singleSetup makes more sense --- src/io/video.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/io/video.js b/src/io/video.js index f8957dd62..b24dff314 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -126,8 +126,8 @@ class Video { * @return {Promise} When video has been received, rejected if video is not received */ _setupVideo () { - if (this._lastSetup) { - return this._lastSetup; + if (this._singleSetup) { + return this._singleSetup; } this._video = document.createElement('video'); @@ -149,17 +149,18 @@ class Video { resolve(this._video); }, err => { // There are probably some error types we could handle gracefully here. - this._lastSetup = null; + this._singleSetup = null; reject(err); }); }); - return video.then(() => this._setupPreview()); + this._singleSetup = video.then(() => this._setupPreview()); + return this._singleSetup; } _disableVideo () { this._disablePreview(); - this._lastSetup = null; + this._singleSetup = null; // by clearing refs to video and track, we should lose our hold over the camera this._video = null; this._track = null; From d51204241526b78366da1d103ed5a791e27e1dee Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 15:43:57 -0400 Subject: [PATCH 04/22] Add set preview ghost --- src/io/video.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/io/video.js b/src/io/video.js index b24dff314..84ece4d12 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -167,8 +167,9 @@ class Video { } _disablePreview () { - if (this._skin) { + if (this._skin && this._drawable) { this._skin.clear(); + this._drawable.updateProperties({visible: false}); } this._renderPreviewFrame = null; } @@ -213,7 +214,16 @@ class Video { this._renderPreviewFrame(); } + } + /** + * Set the preview ghost effect + * @param {number} ghost from 0 (visible) to 100 (invisible) - ghost effect + */ + setPreviewGhost (ghost) { + if (this._drawable) { + this._drawable.updateProperties({ghost}); + } } get videoReady () { From 992e8846689c422ac9c27961a02e2f0f13786a20 Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 16:01:10 -0400 Subject: [PATCH 05/22] Make disableVideo a public method --- src/io/video.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/io/video.js b/src/io/video.js index 84ece4d12..892f19da2 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -158,7 +158,7 @@ class Video { return this._singleSetup; } - _disableVideo () { + disableVideo () { this._disablePreview(); this._singleSetup = null; // by clearing refs to video and track, we should lose our hold over the camera @@ -193,6 +193,8 @@ class Video { // if we haven't already created and started a preview frame render loop, do so if (!this._renderPreviewFrame) { + this._drawable.updateProperties({visible: true}); + this._renderPreviewFrame = () => { if (!this._renderPreviewFrame) { return; From e4bd9cf6b2f58ce2d31b5de6ecc50bb50de06ad3 Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 16:03:48 -0400 Subject: [PATCH 06/22] Move methods around to sort the public interface nearer the top --- src/io/video.js | 293 ++++++++++++++++++++++++------------------------ 1 file changed, 149 insertions(+), 144 deletions(-) diff --git a/src/io/video.js b/src/io/video.js index 892f19da2..1394095b9 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -120,44 +120,8 @@ class Video { } /** - * Create a video stream. - * Should probably be moved to -render or somewhere similar later - * @private - * @return {Promise} When video has been received, rejected if video is not received + * Disable video stream (turn video off) */ - _setupVideo () { - if (this._singleSetup) { - return this._singleSetup; - } - - this._video = document.createElement('video'); - const video = new Promise((resolve, reject) => { - navigator.getUserMedia({ - audio: false, - video: { - width: {min: 480, ideal: 640}, - height: {min: 360, ideal: 480} - } - }, stream => { - this._video.src = window.URL.createObjectURL(stream); - // Hint to the stream that it should load. A standard way to do this - // is add the video tag to the DOM. Since this extension wants to - // hide the video tag and instead render a sample of the stream into - // the webgl rendered Scratch canvas, another hint like this one is - // needed. - this._track = stream.getTracks()[0]; - resolve(this._video); - }, err => { - // There are probably some error types we could handle gracefully here. - this._singleSetup = null; - reject(err); - }); - }); - - this._singleSetup = video.then(() => this._setupPreview()); - return this._singleSetup; - } - disableVideo () { this._disablePreview(); this._singleSetup = null; @@ -166,113 +130,6 @@ class Video { this._track = null; } - _disablePreview () { - if (this._skin && this._drawable) { - this._skin.clear(); - this._drawable.updateProperties({visible: false}); - } - this._renderPreviewFrame = null; - } - - _setupPreview () { - const {renderer} = this.runtime; - if (!renderer) return; - - if (this._skinId === -1 && this._skin === null && this._drawable === -1) { - this._skinId = renderer.createPenSkin(); - this._skin = renderer._allSkins[this._skinId]; - this._drawable = renderer.createDrawable(); - renderer.setDrawableOrder( - this._drawable, - Video.ORDER - ); - renderer.updateDrawableProperties(this._drawable, { - skinId: this._skinId - }); - } - - // if we haven't already created and started a preview frame render loop, do so - if (!this._renderPreviewFrame) { - this._drawable.updateProperties({visible: true}); - - this._renderPreviewFrame = () => { - if (!this._renderPreviewFrame) { - return; - } - - setTimeout(this._renderPreviewFrame, this.runtime.currentStepTime); - - const canvas = this.getFrame({format: Video.FORMAT_CANVAS}); - - if (!canvas) { - return; - } - - const xOffset = Video.DIMENSIONS[0] / -2; - const yOffset = Video.DIMENSIONS[1] / 2; - this._skin.drawStamp(canvas, xOffset, yOffset); - this.runtime.requestRedraw(); - }; - - this._renderPreviewFrame(); - } - } - - /** - * Set the preview ghost effect - * @param {number} ghost from 0 (visible) to 100 (invisible) - ghost effect - */ - setPreviewGhost (ghost) { - if (this._drawable) { - this._drawable.updateProperties({ghost}); - } - } - - get videoReady () { - if (!this._video) { - return false; - } - if (!this._track) { - return false; - } - const {videoWidth, videoHeight} = this._video; - if (typeof videoWidth !== 'number' || typeof videoHeight !== 'number') { - return false; - } - if (videoWidth === 0 || videoHeight === 0) { - return false; - } - return true; - } - - /** - * get an internal workspace for canvas/context/caches - * this uses some document stuff to create a canvas and what not, probably needs abstraction - * into the renderer layer? - * @private - * @return {object} A workspace for canvas/data storage. Internal format not documented intentionally - */ - _getWorkspace ({dimensions, mirror}) { - let workspace = this._workspace.find(space => ( - space.dimensions.join('-') === dimensions.join('-') && - space.mirror === mirror - )); - if (!workspace) { - workspace = { - dimensions, - mirror, - canvas: document.createElement('canvas'), - lastUpdate: 0, - cacheData: {} - }; - workspace.canvas.width = dimensions[0]; - workspace.canvas.height = dimensions[1]; - workspace.context = workspace.canvas.getContext('2d'); - this._workspace.push(workspace); - } - return workspace; - } - /** * Return frame data from the video feed in a specified dimensions, format, and mirroring. * @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors. @@ -337,6 +194,154 @@ class Video { return formatCache.lastData; } + + /** + * Set the preview ghost effect + * @param {number} ghost from 0 (visible) to 100 (invisible) - ghost effect + */ + setPreviewGhost (ghost) { + if (this._drawable) { + this._drawable.updateProperties({ghost}); + } + } + + + /** + * Create a video stream. + * Should probably be moved to -render or somewhere similar later + * @private + * @return {Promise} When video has been received, rejected if video is not received + */ + _setupVideo () { + if (this._singleSetup) { + return this._singleSetup; + } + + this._video = document.createElement('video'); + const video = new Promise((resolve, reject) => { + navigator.getUserMedia({ + audio: false, + video: { + width: {min: 480, ideal: 640}, + height: {min: 360, ideal: 480} + } + }, stream => { + this._video.src = window.URL.createObjectURL(stream); + // Hint to the stream that it should load. A standard way to do this + // is add the video tag to the DOM. Since this extension wants to + // hide the video tag and instead render a sample of the stream into + // the webgl rendered Scratch canvas, another hint like this one is + // needed. + this._track = stream.getTracks()[0]; + resolve(this._video); + }, err => { + // There are probably some error types we could handle gracefully here. + this._singleSetup = null; + reject(err); + }); + }); + + this._singleSetup = video.then(() => this._setupPreview()); + return this._singleSetup; + } + + _disablePreview () { + if (this._skin && this._drawable) { + this._skin.clear(); + this._drawable.updateProperties({visible: false}); + } + this._renderPreviewFrame = null; + } + + _setupPreview () { + const {renderer} = this.runtime; + if (!renderer) return; + + if (this._skinId === -1 && this._skin === null && this._drawable === -1) { + this._skinId = renderer.createPenSkin(); + this._skin = renderer._allSkins[this._skinId]; + this._drawable = renderer.createDrawable(); + renderer.setDrawableOrder( + this._drawable, + Video.ORDER + ); + renderer.updateDrawableProperties(this._drawable, { + skinId: this._skinId + }); + } + + // if we haven't already created and started a preview frame render loop, do so + if (!this._renderPreviewFrame) { + this._drawable.updateProperties({visible: true}); + + this._renderPreviewFrame = () => { + if (!this._renderPreviewFrame) { + return; + } + + setTimeout(this._renderPreviewFrame, this.runtime.currentStepTime); + + const canvas = this.getFrame({format: Video.FORMAT_CANVAS}); + + if (!canvas) { + return; + } + + const xOffset = Video.DIMENSIONS[0] / -2; + const yOffset = Video.DIMENSIONS[1] / 2; + this._skin.drawStamp(canvas, xOffset, yOffset); + this.runtime.requestRedraw(); + }; + + this._renderPreviewFrame(); + } + } + + get videoReady () { + if (!this._video) { + return false; + } + if (!this._track) { + return false; + } + const {videoWidth, videoHeight} = this._video; + if (typeof videoWidth !== 'number' || typeof videoHeight !== 'number') { + return false; + } + if (videoWidth === 0 || videoHeight === 0) { + return false; + } + return true; + } + + /** + * get an internal workspace for canvas/context/caches + * this uses some document stuff to create a canvas and what not, probably needs abstraction + * into the renderer layer? + * @private + * @return {object} A workspace for canvas/data storage. Internal format not documented intentionally + */ + _getWorkspace ({dimensions, mirror}) { + let workspace = this._workspace.find(space => ( + space.dimensions.join('-') === dimensions.join('-') && + space.mirror === mirror + )); + if (!workspace) { + workspace = { + dimensions, + mirror, + canvas: document.createElement('canvas'), + lastUpdate: 0, + cacheData: {} + }; + workspace.canvas.width = dimensions[0]; + workspace.canvas.height = dimensions[1]; + workspace.context = workspace.canvas.getContext('2d'); + this._workspace.push(workspace); + } + return workspace; + } + } From 90166dc73221dff2be18d79dedcf4b2d64839d5b Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 17:46:31 -0400 Subject: [PATCH 07/22] Add extra control blocks for video transparency and on/off control --- .../scratch3_video_sensing/index.js | 52 ++++++++++++++++--- src/io/video.js | 8 +-- src/serialization/sb2_specmap.js | 8 +-- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 3a276768a..58796cf0e 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -41,11 +41,7 @@ class Scratch3VideoSensingBlocks { // Boot up the video, canvas to down/up sample the video stream, the // preview skin and drawable, and kick off looping the analysis // logic. - this.runtime.ioDevices.video.requestVideo() - .then(({release}) => { - this.releaseVideo = release; - this._loop(); - }); + this._loop(); } } @@ -218,6 +214,14 @@ class Scratch3VideoSensingBlocks { return 2; } + get VIDEO_STATE_INFO () { + return [ + {name: 'off'}, + {name: 'on'}, + {name: 'on-flipped'} + ]; + } + /** * @returns {object} metadata for this extension and its blocks. */ @@ -255,11 +259,33 @@ class Scratch3VideoSensingBlocks { defaultValue: 10 } } + }, + { + opcode: 'videoToggle', + text: 'turn video [VIDEO_STATE]', + arguments: { + VIDEO_STATE: { + type: ArgumentType.NUMBER, + menu: 'VIDEO_STATE', + defaultValue: 1 + } + } + }, + { + opcode: 'setVideoTransparency', + text: 'set video transparency to [TRANSPARENCY]', + arguments: { + TRANSPARENCY: { + type: ArgumentType.NUMBER, + defaultValue: 0 + } + } } ], menus: { MOTION_DIRECTION: this._buildMenu(this.MOTION_DIRECTION_INFO), - STAGE_SPRITE: this._buildMenu(this.STAGE_SPRITE_INFO) + STAGE_SPRITE: this._buildMenu(this.STAGE_SPRITE_INFO), + VIDEO_STATE: this._buildMenu(this.VIDEO_STATE_INFO) } }; } @@ -312,6 +338,20 @@ class Scratch3VideoSensingBlocks { const state = this._analyzeLocalMotion(util.target); return state.motionAmount > Number(args.REFERENCE); } + + videoToggle (args) { + const state = Number(args.VIDEO_STATE); + // 1 == off, 2 & 3 are on (3 is flipped) + if (state > 1) { + this.runtime.ioDevices.video.requestVideo(); + } else { + this.runtime.ioDevices.video.disableVideo(); + } + } + + setVideoTransparency (args) { + this.runtime.ioDevices.video.setPreviewGhost(Number(args.TRANSPARENCY)); + } } module.exports = Scratch3VideoSensingBlocks; diff --git a/src/io/video.js b/src/io/video.js index 1394095b9..c1888e1ac 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -201,7 +201,7 @@ class Video { */ setPreviewGhost (ghost) { if (this._drawable) { - this._drawable.updateProperties({ghost}); + this.runtime.renderer.updateDrawableProperties(this._drawable, {ghost}); } } @@ -248,7 +248,7 @@ class Video { _disablePreview () { if (this._skin && this._drawable) { this._skin.clear(); - this._drawable.updateProperties({visible: false}); + this.runtime.renderer.updateDrawableProperties(this._drawable, {visible: false}); } this._renderPreviewFrame = null; } @@ -272,7 +272,9 @@ class Video { // if we haven't already created and started a preview frame render loop, do so if (!this._renderPreviewFrame) { - this._drawable.updateProperties({visible: true}); + renderer.updateDrawableProperties(this._drawable, { + visible: true + }); this._renderPreviewFrame = () => { if (!this._renderPreviewFrame) { diff --git a/src/serialization/sb2_specmap.js b/src/serialization/sb2_specmap.js index e5bb65b62..b9a07ddae 100644 --- a/src/serialization/sb2_specmap.js +++ b/src/serialization/sb2_specmap.js @@ -980,17 +980,17 @@ const specMap = { // ] // }, 'setVideoState': { - opcode: 'sensing_videotoggle', + opcode: 'videoSensing.videoToggle', argMap: [ { type: 'input', - inputOp: 'sensing_videotogglemenu', - inputName: 'VIDEOTOGGLEMENU' + inputOp: 'videoSensing.menu.VIDEO_STATE', + inputName: 'VIDEO_STATE' } ] }, 'setVideoTransparency': { - opcode: 'sensing_setvideotransparency', + opcode: 'videoSensing.setVideoTransparency', argMap: [ { type: 'input', From 46b4ef4d805a747b5200776230cb31b716711e60 Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 18:09:05 -0400 Subject: [PATCH 08/22] Use a global mirror state --- src/extensions/scratch3_video_sensing/index.js | 9 +++++---- src/io/video.js | 8 +++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 58796cf0e..145a204d4 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -117,7 +117,6 @@ class Scratch3VideoSensingBlocks { const offset = time - this._lastUpdate; if (offset > Scratch3VideoSensingBlocks.INTERVAL) { const frame = this.runtime.ioDevices.video.getFrame({ - mirror: true, format: Video.FORMAT_IMAGE_DATA, dimensions: Scratch3VideoSensingBlocks.DIMENSIONS }); @@ -340,12 +339,14 @@ class Scratch3VideoSensingBlocks { } videoToggle (args) { + // imported blocks have VIDEO_STATE "off", "on", "on-flipped" as opposed to the numerics? const state = Number(args.VIDEO_STATE); // 1 == off, 2 & 3 are on (3 is flipped) - if (state > 1) { - this.runtime.ioDevices.video.requestVideo(); - } else { + if (args.VIDEO_STATE === 'off' || state === 1) { this.runtime.ioDevices.video.disableVideo(); + } else { + this.runtime.ioDevices.video.requestVideo(); + this.runtime.ioDevices.video.mirror = args.VIDEO_STATE === 'on' || state === 2; } } diff --git a/src/io/video.js b/src/io/video.js index c1888e1ac..f5e36a56c 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -8,6 +8,12 @@ class Video { */ this.runtime = runtime; + /** + * Default value for mirrored frames. + * @type boolean + */ + this.mirror = true; + /** * Cache frames for this many ms. * @type number @@ -136,7 +142,7 @@ class Video { */ getFrame ({ dimensions = Video.DIMENSIONS, - mirror = true, + mirror = this.mirror, format = Video.FORMAT_IMAGE_DATA, cacheTimeout = this._frameCacheTimeout }) { From efa6b3a0698ea4e5dbd334596f48a0174ccc44e9 Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Tue, 3 Apr 2018 18:16:12 -0400 Subject: [PATCH 09/22] stop track --- src/io/video.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/io/video.js b/src/io/video.js index f5e36a56c..545902f4e 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -133,6 +133,9 @@ class Video { this._singleSetup = null; // by clearing refs to video and track, we should lose our hold over the camera this._video = null; + if (this._track) { + this._track.stop(); + } this._track = null; } From f19ae793c0f2a3ab4c53e26889f1043098e22f2a Mon Sep 17 00:00:00 2001 From: Corey Frang Date: Wed, 4 Apr 2018 14:20:13 -0400 Subject: [PATCH 10/22] enableVideo - get rid of "requests" --- .../scratch3_video_sensing/index.js | 2 +- src/io/video.js | 35 +++---------------- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/src/extensions/scratch3_video_sensing/index.js b/src/extensions/scratch3_video_sensing/index.js index 145a204d4..5fca73e53 100644 --- a/src/extensions/scratch3_video_sensing/index.js +++ b/src/extensions/scratch3_video_sensing/index.js @@ -345,7 +345,7 @@ class Scratch3VideoSensingBlocks { if (args.VIDEO_STATE === 'off' || state === 1) { this.runtime.ioDevices.video.disableVideo(); } else { - this.runtime.ioDevices.video.requestVideo(); + this.runtime.ioDevices.video.enableVideo(); this.runtime.ioDevices.video.mirror = args.VIDEO_STATE === 'on' || state === 2; } } diff --git a/src/io/video.js b/src/io/video.js index 545902f4e..da7e462d9 100644 --- a/src/io/video.js +++ b/src/io/video.js @@ -20,12 +20,6 @@ class Video { */ this._frameCacheTimeout = 16; - /** - * Store each request for video, so when all are released we can disable preview/video feed. - * @type Array. - */ - this._requests = []; - /** * DOM Video element * @private @@ -93,36 +87,15 @@ class Video { * Request video be enabled. Sets up video, creates video skin and enables preview. * * ioDevices.video.requestVideo() - * .then(({ release }) => { - * this.releaseVideo = release; - * }) * - * @return {Promise.} A request object with a "release" property that - * should be called when you are done with the video. + * @return {Promise.