From 28f90648b04b652d68bede05b1d1992ffc1dea92 Mon Sep 17 00:00:00 2001 From: Ray Schamp Date: Fri, 22 Jun 2018 09:27:23 -0400 Subject: [PATCH] Revert #1260, #1258, #1239 This reverts commits c268bbae47ce5e86c47894b2d813645820793590 61dacfc915bf7a6b385c20516579bd19c13016ec 30b8cb8eb1ca1c7af5d1afb079d7176c6cac8273 --- src/blocks/scratch3_sound.js | 47 ++++---- src/extensions/scratch3_music/index.js | 151 +++++++++++-------------- src/import/load-sound.js | 20 ++-- src/serialization/sb3.js | 2 +- src/sprites/rendered-target.js | 43 +++---- src/sprites/sprite.js | 14 +-- src/virtual-machine.js | 6 +- test/unit/blocks_sounds.js | 8 +- 8 files changed, 124 insertions(+), 167 deletions(-) diff --git a/src/blocks/scratch3_sound.js b/src/blocks/scratch3_sound.js index 16b041bf8..a599b314c 100644 --- a/src/blocks/scratch3_sound.js +++ b/src/blocks/scratch3_sound.js @@ -88,7 +88,6 @@ class Scratch3SoundBlocks { if (!soundState) { soundState = Clone.simple(Scratch3SoundBlocks.DEFAULT_SOUND_STATE); target.setCustomState(Scratch3SoundBlocks.STATE_KEY, soundState); - target.soundEffects = soundState.effects; } return soundState; } @@ -140,19 +139,20 @@ class Scratch3SoundBlocks { } playSound (args, util) { - // Don't return the promise, it's the only difference for AndWait - this.playSoundAndWait(args, util); + const index = this._getSoundIndex(args.SOUND_MENU, util); + if (index >= 0) { + const soundId = util.target.sprite.sounds[index].soundId; + if (util.target.audioPlayer === null) return; + util.target.audioPlayer.playSound(soundId); + } } playSoundAndWait (args, util) { const index = this._getSoundIndex(args.SOUND_MENU, util); if (index >= 0) { - const {target} = util; - const {sprite} = target; - const {soundId} = sprite.sounds[index]; - if (sprite.soundBank) { - return sprite.soundBank.playSound(target, soundId); - } + const soundId = util.target.sprite.sounds[index].soundId; + if (util.target.audioPlayer === null) return; + return util.target.audioPlayer.playSound(soundId); } } @@ -199,9 +199,8 @@ class Scratch3SoundBlocks { } _stopAllSoundsForTarget (target) { - if (target.sprite.soundBank) { - target.sprite.soundBank.stopAllSounds(target); - } + if (target.audioPlayer === null) return; + target.audioPlayer.stopAllSounds(); } setEffect (args, util) { @@ -225,19 +224,23 @@ class Scratch3SoundBlocks { soundState.effects[effect] = value; } - const {min, max} = Scratch3SoundBlocks.EFFECT_RANGE[effect]; - soundState.effects[effect] = MathUtil.clamp(soundState.effects[effect], min, max); + const effectRange = Scratch3SoundBlocks.EFFECT_RANGE[effect]; + soundState.effects[effect] = MathUtil.clamp(soundState.effects[effect], effectRange.min, effectRange.max); + + if (util.target.audioPlayer === null) return; + util.target.audioPlayer.setEffect(effect, soundState.effects[effect]); - this._syncEffectsForTarget(util.target); // Yield until the next tick. return Promise.resolve(); } _syncEffectsForTarget (target) { - if (!target || !target.sprite.soundBank) return; - target.soundEffects = this._getSoundState(target).effects; - - target.sprite.soundBank.setEffects(target); + if (!target || !target.audioPlayer) return; + const soundState = this._getSoundState(target); + for (const effect in soundState.effects) { + if (!soundState.effects.hasOwnProperty(effect)) continue; + target.audioPlayer.setEffect(effect, soundState.effects[effect]); + } } clearEffects (args, util) { @@ -250,7 +253,8 @@ class Scratch3SoundBlocks { if (!soundState.effects.hasOwnProperty(effect)) continue; soundState.effects[effect] = 0; } - this._syncEffectsForTarget(target); + if (target.audioPlayer === null) return; + target.audioPlayer.clearEffects(); } _clearEffectsForAllTargets () { @@ -274,7 +278,8 @@ class Scratch3SoundBlocks { _updateVolume (volume, util) { volume = MathUtil.clamp(volume, 0, 100); util.target.volume = volume; - this._syncEffectsForTarget(util.target); + if (util.target.audioPlayer === null) return; + util.target.audioPlayer.setVolume(util.target.volume); // Yield until the next tick. return Promise.resolve(); diff --git a/src/extensions/scratch3_music/index.js b/src/extensions/scratch3_music/index.js index 9099dc98a..852c9261a 100644 --- a/src/extensions/scratch3_music/index.js +++ b/src/extensions/scratch3_music/index.js @@ -52,25 +52,18 @@ class Scratch3MusicBlocks { this._concurrencyCounter = 0; /** - * An array of sound players, one for each drum sound. + * An array of audio buffers, one for each drum sound. * @type {Array} * @private */ - this._drumPlayers = []; + this._drumBuffers = []; /** - * An array of arrays of sound players. Each instrument has one or more audio players. + * An array of arrays of audio buffers. Each instrument has one or more audio buffers. * @type {Array[]} * @private */ - this._instrumentPlayerArrays = []; - - /** - * An array of arrays of sound players. Each instrument mya have an audio player for each playable note. - * @type {Array[]} - * @private - */ - this._instrumentPlayerNoteArrays = []; + this._instrumentBufferArrays = []; /** * An array of audio bufferSourceNodes. Each time you play an instrument or drum sound, @@ -94,15 +87,14 @@ class Scratch3MusicBlocks { const loadingPromises = []; this.DRUM_INFO.forEach((drumInfo, index) => { const filePath = `drums/${drumInfo.fileName}`; - const promise = this._storeSound(filePath, index, this._drumPlayers); + const promise = this._storeSound(filePath, index, this._drumBuffers); loadingPromises.push(promise); }); this.INSTRUMENT_INFO.forEach((instrumentInfo, instrumentIndex) => { - this._instrumentPlayerArrays[instrumentIndex] = []; - this._instrumentPlayerNoteArrays[instrumentIndex] = []; + this._instrumentBufferArrays[instrumentIndex] = []; instrumentInfo.samples.forEach((sample, noteIndex) => { const filePath = `instruments/${instrumentInfo.dirName}/${sample}`; - const promise = this._storeSound(filePath, noteIndex, this._instrumentPlayerArrays[instrumentIndex]); + const promise = this._storeSound(filePath, noteIndex, this._instrumentBufferArrays[instrumentIndex]); loadingPromises.push(promise); }); }); @@ -112,22 +104,22 @@ class Scratch3MusicBlocks { } /** - * Decode a sound and store the player in an array. + * Decode a sound and store the buffer in an array. * @param {string} filePath - the audio file name. - * @param {number} index - the index at which to store the audio player. - * @param {array} playerArray - the array of players in which to store it. + * @param {number} index - the index at which to store the audio buffer. + * @param {array} bufferArray - the array of buffers in which to store it. * @return {Promise} - a promise which will resolve once the sound has been stored. */ - _storeSound (filePath, index, playerArray) { + _storeSound (filePath, index, bufferArray) { const fullPath = `${filePath}.mp3`; if (!assetData[fullPath]) return; - // The sound player has already been downloaded via the manifest file required above. + // The sound buffer has already been downloaded via the manifest file required above. const soundBuffer = assetData[fullPath]; - return this._decodeSound(soundBuffer).then(player => { - playerArray[index] = player; + return this._decodeSound(soundBuffer).then(buffer => { + bufferArray[index] = buffer; }); } @@ -137,14 +129,24 @@ class Scratch3MusicBlocks { * @return {Promise} - a promise which will resolve once the sound has decoded. */ _decodeSound (soundBuffer) { - const engine = this.runtime.audioEngine; + const context = this.runtime.audioEngine && this.runtime.audioEngine.audioContext; - if (!engine) { + if (!context) { return Promise.reject(new Error('No Audio Context Detected')); } // Check for newer promise-based API - return engine.decodeSoundPlayer({data: {buffer: soundBuffer}}); + if (context.decodeAudioData.length === 1) { + return context.decodeAudioData(soundBuffer); + } else { // eslint-disable-line no-else-return + // Fall back to callback API + return new Promise((resolve, reject) => + context.decodeAudioData(soundBuffer, + buffer => resolve(buffer), + error => reject(error) + ) + ); + } } /** @@ -776,34 +778,26 @@ class Scratch3MusicBlocks { */ _playDrumNum (util, drumNum) { if (util.runtime.audioEngine === null) return; - if (util.target.sprite.soundBank === null) return; + if (util.target.audioPlayer === null) return; // If we're playing too many sounds, do not play the drum sound. if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) { return; } + const outputNode = util.target.audioPlayer.getInputNode(); + const context = util.runtime.audioEngine.audioContext; + const bufferSource = context.createBufferSource(); + bufferSource.buffer = this._drumBuffers[drumNum]; + bufferSource.connect(outputNode); + bufferSource.start(); - const player = this._drumPlayers[drumNum]; - - if (typeof player === 'undefined') return; - - if (player.isPlaying) { - // Take the internal player state and create a new player with it. - // `.play` does this internally but then instructs the sound to - // stop. - player.take(); - } - - const engine = util.runtime.audioEngine; - const chain = engine.createEffectChain(); - chain.setEffectsFromTarget(util.target); - player.connect(chain); + const bufferSourceIndex = this._bufferSources.length; + this._bufferSources.push(bufferSource); this._concurrencyCounter++; - player.once('stop', () => { + bufferSource.onended = () => { this._concurrencyCounter--; - }); - - player.play(); + delete this._bufferSources[bufferSourceIndex]; + }; } /** @@ -862,7 +856,7 @@ class Scratch3MusicBlocks { */ _playNote (util, note, durationSec) { if (util.runtime.audioEngine === null) return; - if (util.target.sprite.soundBank === null) return; + if (util.target.audioPlayer === null) return; // If we're playing too many sounds, do not play the note. if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) { @@ -877,37 +871,28 @@ class Scratch3MusicBlocks { const sampleIndex = this._selectSampleIndexForNote(note, sampleArray); // If the audio sample has not loaded yet, bail out - if (typeof this._instrumentPlayerArrays[inst] === 'undefined') return; - if (typeof this._instrumentPlayerArrays[inst][sampleIndex] === 'undefined') return; + if (typeof this._instrumentBufferArrays[inst] === 'undefined') return; + if (typeof this._instrumentBufferArrays[inst][sampleIndex] === 'undefined') return; - // Fetch the sound player to play the note. - const engine = util.runtime.audioEngine; + // Create the audio buffer to play the note, and set its pitch + const context = util.runtime.audioEngine.audioContext; + const bufferSource = context.createBufferSource(); - if (!this._instrumentPlayerNoteArrays[inst][note]) { - this._instrumentPlayerNoteArrays[inst][note] = this._instrumentPlayerArrays[inst][sampleIndex].take(); - } + const bufferSourceIndex = this._bufferSources.length; + this._bufferSources.push(bufferSource); - const player = this._instrumentPlayerNoteArrays[inst][note]; - - if (player.isPlaying) { - // Take the internal player state and create a new player with it. - // `.play` does this internally but then instructs the sound to - // stop. - player.take(); - } - - const chain = engine.createEffectChain(); - chain.setEffectsFromTarget(util.target); - - // Set its pitch. + bufferSource.buffer = this._instrumentBufferArrays[inst][sampleIndex]; const sampleNote = sampleArray[sampleIndex]; - const notePitchInterval = this._ratioForPitchInterval(note - sampleNote); + bufferSource.playbackRate.value = this._ratioForPitchInterval(note - sampleNote); - // Create a gain node for this note, and connect it to the sprite's - // simulated effectChain. - const context = engine.audioContext; - const releaseGain = context.createGain(); - releaseGain.connect(chain.getInputNode()); + // Create a gain node for this note, and connect it to the sprite's audioPlayer. + const gainNode = context.createGain(); + bufferSource.connect(gainNode); + const outputNode = util.target.audioPlayer.getInputNode(); + gainNode.connect(outputNode); + + // Start playing the note + bufferSource.start(); // Schedule the release of the note, ramping its gain down to zero, // and then stopping the sound. @@ -917,24 +902,16 @@ class Scratch3MusicBlocks { } const releaseStart = context.currentTime + durationSec; const releaseEnd = releaseStart + releaseDuration; - releaseGain.gain.setValueAtTime(1, releaseStart); - releaseGain.gain.linearRampToValueAtTime(0.0001, releaseEnd); + gainNode.gain.setValueAtTime(1, releaseStart); + gainNode.gain.linearRampToValueAtTime(0.0001, releaseEnd); + bufferSource.stop(releaseEnd); + // Update the concurrency counter this._concurrencyCounter++; - player.once('stop', () => { + bufferSource.onended = () => { this._concurrencyCounter--; - }); - - // Start playing the note - player.play(); - // Connect the player to the gain node. - player.connect({getInputNode () { - return releaseGain; - }}); - // Set playback now after play creates the outputNode. - player.outputNode.playbackRate.value = notePitchInterval; - // Schedule playback to stop. - player.outputNode.stop(releaseEnd); + delete this._bufferSources[bufferSourceIndex]; + }; } /** diff --git a/src/import/load-sound.js b/src/import/load-sound.js index 8f4f7a1a6..e13db14d2 100644 --- a/src/import/load-sound.js +++ b/src/import/load-sound.js @@ -8,32 +8,27 @@ const log = require('../util/log'); * @property {Buffer} data - sound data will be written here once loaded. * @param {!Asset} soundAsset - the asset loaded from storage. * @param {!Runtime} runtime - Scratch runtime, used to access the storage module. - * @param {Sprite} sprite - Scratch sprite to add sounds to. * @returns {!Promise} - a promise which will resolve to the sound when ready. */ -const loadSoundFromAsset = function (sound, soundAsset, runtime, sprite) { +const loadSoundFromAsset = function (sound, soundAsset, runtime) { sound.assetId = soundAsset.assetId; if (!runtime.audioEngine) { log.error('No audio engine present; cannot load sound asset: ', sound.md5); return Promise.resolve(sound); } - return runtime.audioEngine.decodeSoundPlayer(Object.assign( + return runtime.audioEngine.decodeSound(Object.assign( {}, sound, {data: soundAsset.data} - )).then(soundPlayer => { - sound.soundId = soundPlayer.id; + )).then(soundId => { + sound.soundId = soundId; // Set the sound sample rate and sample count based on the // the audio buffer from the audio engine since the sound // gets resampled by the audio engine - const soundBuffer = soundPlayer.buffer; + const soundBuffer = runtime.audioEngine.getSoundBuffer(soundId); sound.rate = soundBuffer.sampleRate; sound.sampleCount = soundBuffer.length; - if (sprite.soundBank !== null) { - sprite.soundBank.addSoundPlayer(soundPlayer); - } - return sound; }); }; @@ -44,10 +39,9 @@ const loadSoundFromAsset = function (sound, soundAsset, runtime, sprite) { * @property {string} md5 - the MD5 and extension of the sound to be loaded. * @property {Buffer} data - sound data will be written here once loaded. * @param {!Runtime} runtime - Scratch runtime, used to access the storage module. - * @param {Sprite} sprite - Scratch sprite to add sounds to. * @returns {!Promise} - a promise which will resolve to the sound when ready. */ -const loadSound = function (sound, runtime, sprite) { +const loadSound = function (sound, runtime) { if (!runtime.storage) { log.error('No storage module present; cannot load sound asset: ', sound.md5); return Promise.resolve(sound); @@ -58,7 +52,7 @@ const loadSound = function (sound, runtime, sprite) { return runtime.storage.load(runtime.storage.AssetType.Sound, md5, ext) .then(soundAsset => { sound.dataFormat = ext; - return loadSoundFromAsset(sound, soundAsset, runtime, sprite); + return loadSoundFromAsset(sound, soundAsset, runtime); }); }; diff --git a/src/serialization/sb3.js b/src/serialization/sb3.js index 0ab76a15d..4f129c171 100644 --- a/src/serialization/sb3.js +++ b/src/serialization/sb3.js @@ -800,7 +800,7 @@ const parseScratchObject = function (object, runtime, extensions, zip) { // any translation that needs to happen will happen in the process // of building up the costume object into an sb3 format return deserializeSound(sound, runtime, zip) - .then(() => loadSound(sound, runtime, sprite)); + .then(() => loadSound(sound, runtime)); // Only attempt to load the sound after the deserialization // process has been completed. }); diff --git a/src/sprites/rendered-target.js b/src/sprites/rendered-target.js index 93e7d8c7f..aac0b059a 100644 --- a/src/sprites/rendered-target.js +++ b/src/sprites/rendered-target.js @@ -170,30 +170,21 @@ class RenderedTarget extends Target { } } - get audioPlayer () { - /* eslint-disable no-console */ - console.warn('get audioPlayer deprecated, please update to use .sprite.soundBank methods'); - console.warn(new Error('stack for debug').stack); - /* eslint-enable no-console */ - const bank = this.sprite.soundBank; - const audioPlayerProxy = { - playSound: soundId => bank.play(this, soundId) - }; - - Object.defineProperty(this, 'audioPlayer', { - configurable: false, - enumerable: true, - writable: false, - value: audioPlayerProxy - }); - - return audioPlayerProxy; - } - /** * Initialize the audio player for this sprite or clone. */ initAudio () { + this.audioPlayer = null; + if (this.runtime && this.runtime.audioEngine) { + this.audioPlayer = this.runtime.audioEngine.createPlayer(); + // If this is a clone, it gets a reference to its parent's activeSoundPlayers object. + if (!this.isOriginal) { + const parent = this.sprite.clones[0]; + if (parent && parent.audioPlayer) { + this.audioPlayer.activeSoundPlayers = parent.audioPlayer.activeSoundPlayers; + } + } + } } /** @@ -1043,8 +1034,9 @@ class RenderedTarget extends Target { */ onStopAll () { this.clearEffects(); - if (this.sprite.soundBank) { - this.sprite.soundBank.stopAllSounds(this); + if (this.audioPlayer) { + this.audioPlayer.stopAllSounds(); + this.audioPlayer.clearEffects(); } } @@ -1130,9 +1122,6 @@ class RenderedTarget extends Target { dispose () { this.runtime.changeCloneCounter(-1); this.runtime.stopForTarget(this); - if (this.sprite.soundBank) { - this.sprite.soundBank.stopAllSounds(this); - } this.sprite.removeClone(this); if (this.renderer && this.drawableID !== null) { this.renderer.destroyDrawable(this.drawableID, this.isStage ? @@ -1143,6 +1132,10 @@ class RenderedTarget extends Target { this.runtime.requestRedraw(); } } + if (this.audioPlayer) { + this.audioPlayer.stopAllSounds(); + this.audioPlayer.dispose(); + } } } diff --git a/src/sprites/sprite.js b/src/sprites/sprite.js index c7c35c9e0..de735ecc3 100644 --- a/src/sprites/sprite.js +++ b/src/sprites/sprite.js @@ -8,8 +8,7 @@ const StageLayering = require('../engine/stage-layering'); class Sprite { /** * Sprite to be used on the Scratch stage. - * All clones of a sprite have shared blocks, shared costumes, shared variables, - * shared sounds, etc. + * All clones of a sprite have shared blocks, shared costumes, shared variables. * @param {?Blocks} blocks Shared blocks object for all clones of sprite. * @param {Runtime} runtime Reference to the runtime. * @constructor @@ -48,11 +47,6 @@ class Sprite { * @type {Array.} */ this.clones = []; - - this.soundBank = null; - if (this.runtime && this.runtime.audioEngine) { - this.soundBank = this.runtime.audioEngine.createBank(); - } } /** @@ -161,12 +155,6 @@ class Sprite { return Promise.all(assetPromises).then(() => newSprite); } - - dispose () { - if (this.soundBank) { - this.soundBank.dispose(); - } - } } module.exports = Sprite; diff --git a/src/virtual-machine.js b/src/virtual-machine.js index 9e93b79d2..767641eec 100644 --- a/src/virtual-machine.js +++ b/src/virtual-machine.js @@ -525,7 +525,7 @@ class VirtualMachine extends EventEmitter { * @returns {?Promise} - a promise that resolves when the sound has been decoded and added */ addSound (soundObject) { - return loadSound(soundObject, this.runtime, this.editingTarget.sprite).then(() => { + return loadSound(soundObject, this.runtime).then(() => { this.editingTarget.addSound(soundObject); this.emitTargetsUpdate(); }); @@ -549,7 +549,7 @@ class VirtualMachine extends EventEmitter { getSoundBuffer (soundIndex) { const id = this.editingTarget.sprite.sounds[soundIndex].soundId; if (id && this.runtime && this.runtime.audioEngine) { - return this.editingTarget.sprite.soundBank.getSoundPlayer(id).buffer; + return this.runtime.audioEngine.getSoundBuffer(id); } return null; } @@ -564,7 +564,7 @@ class VirtualMachine extends EventEmitter { const sound = this.editingTarget.sprite.sounds[soundIndex]; const id = sound ? sound.soundId : null; if (id && this.runtime && this.runtime.audioEngine) { - this.editingTarget.sprite.soundBank.getSoundPlayer(id).buffer = newBuffer; + this.runtime.audioEngine.updateSoundBuffer(id, newBuffer); } // Update sound in runtime if (soundEncoding) { diff --git a/test/unit/blocks_sounds.js b/test/unit/blocks_sounds.js index 53acc8aff..9a44e7f9f 100644 --- a/test/unit/blocks_sounds.js +++ b/test/unit/blocks_sounds.js @@ -11,10 +11,10 @@ const util = { {name: 'second name', soundId: 'second soundId'}, {name: 'third name', soundId: 'third soundId'}, {name: '6', soundId: 'fourth soundId'} - ], - soundBank: { - playSound: (target, soundId) => (playedSound = soundId) - } + ] + }, + audioPlayer: { + playSound: soundId => (playedSound = soundId) } } };