diff --git a/src/blocks/scratch3_sound.js b/src/blocks/scratch3_sound.js index a599b314c..16b041bf8 100644 --- a/src/blocks/scratch3_sound.js +++ b/src/blocks/scratch3_sound.js @@ -88,6 +88,7 @@ class Scratch3SoundBlocks { if (!soundState) { soundState = Clone.simple(Scratch3SoundBlocks.DEFAULT_SOUND_STATE); target.setCustomState(Scratch3SoundBlocks.STATE_KEY, soundState); + target.soundEffects = soundState.effects; } return soundState; } @@ -139,20 +140,19 @@ class Scratch3SoundBlocks { } playSound (args, util) { - const index = this._getSoundIndex(args.SOUND_MENU, util); - if (index >= 0) { - const soundId = util.target.sprite.sounds[index].soundId; - if (util.target.audioPlayer === null) return; - util.target.audioPlayer.playSound(soundId); - } + // Don't return the promise, it's the only difference for AndWait + this.playSoundAndWait(args, util); } playSoundAndWait (args, util) { const index = this._getSoundIndex(args.SOUND_MENU, util); if (index >= 0) { - const soundId = util.target.sprite.sounds[index].soundId; - if (util.target.audioPlayer === null) return; - return util.target.audioPlayer.playSound(soundId); + const {target} = util; + const {sprite} = target; + const {soundId} = sprite.sounds[index]; + if (sprite.soundBank) { + return sprite.soundBank.playSound(target, soundId); + } } } @@ -199,8 +199,9 @@ class Scratch3SoundBlocks { } _stopAllSoundsForTarget (target) { - if (target.audioPlayer === null) return; - target.audioPlayer.stopAllSounds(); + if (target.sprite.soundBank) { + target.sprite.soundBank.stopAllSounds(target); + } } setEffect (args, util) { @@ -224,23 +225,19 @@ class Scratch3SoundBlocks { soundState.effects[effect] = value; } - const effectRange = Scratch3SoundBlocks.EFFECT_RANGE[effect]; - soundState.effects[effect] = MathUtil.clamp(soundState.effects[effect], effectRange.min, effectRange.max); - - if (util.target.audioPlayer === null) return; - util.target.audioPlayer.setEffect(effect, soundState.effects[effect]); + const {min, max} = Scratch3SoundBlocks.EFFECT_RANGE[effect]; + soundState.effects[effect] = MathUtil.clamp(soundState.effects[effect], min, max); + this._syncEffectsForTarget(util.target); // Yield until the next tick. return Promise.resolve(); } _syncEffectsForTarget (target) { - if (!target || !target.audioPlayer) return; - const soundState = this._getSoundState(target); - for (const effect in soundState.effects) { - if (!soundState.effects.hasOwnProperty(effect)) continue; - target.audioPlayer.setEffect(effect, soundState.effects[effect]); - } + if (!target || !target.sprite.soundBank) return; + target.soundEffects = this._getSoundState(target).effects; + + target.sprite.soundBank.setEffects(target); } clearEffects (args, util) { @@ -253,8 +250,7 @@ class Scratch3SoundBlocks { if (!soundState.effects.hasOwnProperty(effect)) continue; soundState.effects[effect] = 0; } - if (target.audioPlayer === null) return; - target.audioPlayer.clearEffects(); + this._syncEffectsForTarget(target); } _clearEffectsForAllTargets () { @@ -278,8 +274,7 @@ class Scratch3SoundBlocks { _updateVolume (volume, util) { volume = MathUtil.clamp(volume, 0, 100); util.target.volume = volume; - if (util.target.audioPlayer === null) return; - util.target.audioPlayer.setVolume(util.target.volume); + this._syncEffectsForTarget(util.target); // Yield until the next tick. return Promise.resolve(); diff --git a/src/extensions/scratch3_music/index.js b/src/extensions/scratch3_music/index.js index 852c9261a..9099dc98a 100644 --- a/src/extensions/scratch3_music/index.js +++ b/src/extensions/scratch3_music/index.js @@ -52,18 +52,25 @@ class Scratch3MusicBlocks { this._concurrencyCounter = 0; /** - * An array of audio buffers, one for each drum sound. + * An array of sound players, one for each drum sound. * @type {Array} * @private */ - this._drumBuffers = []; + this._drumPlayers = []; /** - * An array of arrays of audio buffers. Each instrument has one or more audio buffers. + * An array of arrays of sound players. Each instrument has one or more audio players. * @type {Array[]} * @private */ - this._instrumentBufferArrays = []; + this._instrumentPlayerArrays = []; + + /** + * An array of arrays of sound players. Each instrument mya have an audio player for each playable note. + * @type {Array[]} + * @private + */ + this._instrumentPlayerNoteArrays = []; /** * An array of audio bufferSourceNodes. Each time you play an instrument or drum sound, @@ -87,14 +94,15 @@ class Scratch3MusicBlocks { const loadingPromises = []; this.DRUM_INFO.forEach((drumInfo, index) => { const filePath = `drums/${drumInfo.fileName}`; - const promise = this._storeSound(filePath, index, this._drumBuffers); + const promise = this._storeSound(filePath, index, this._drumPlayers); loadingPromises.push(promise); }); this.INSTRUMENT_INFO.forEach((instrumentInfo, instrumentIndex) => { - this._instrumentBufferArrays[instrumentIndex] = []; + this._instrumentPlayerArrays[instrumentIndex] = []; + this._instrumentPlayerNoteArrays[instrumentIndex] = []; instrumentInfo.samples.forEach((sample, noteIndex) => { const filePath = `instruments/${instrumentInfo.dirName}/${sample}`; - const promise = this._storeSound(filePath, noteIndex, this._instrumentBufferArrays[instrumentIndex]); + const promise = this._storeSound(filePath, noteIndex, this._instrumentPlayerArrays[instrumentIndex]); loadingPromises.push(promise); }); }); @@ -104,22 +112,22 @@ class Scratch3MusicBlocks { } /** - * Decode a sound and store the buffer in an array. + * Decode a sound and store the player in an array. * @param {string} filePath - the audio file name. - * @param {number} index - the index at which to store the audio buffer. - * @param {array} bufferArray - the array of buffers in which to store it. + * @param {number} index - the index at which to store the audio player. + * @param {array} playerArray - the array of players in which to store it. * @return {Promise} - a promise which will resolve once the sound has been stored. */ - _storeSound (filePath, index, bufferArray) { + _storeSound (filePath, index, playerArray) { const fullPath = `${filePath}.mp3`; if (!assetData[fullPath]) return; - // The sound buffer has already been downloaded via the manifest file required above. + // The sound player has already been downloaded via the manifest file required above. const soundBuffer = assetData[fullPath]; - return this._decodeSound(soundBuffer).then(buffer => { - bufferArray[index] = buffer; + return this._decodeSound(soundBuffer).then(player => { + playerArray[index] = player; }); } @@ -129,24 +137,14 @@ class Scratch3MusicBlocks { * @return {Promise} - a promise which will resolve once the sound has decoded. */ _decodeSound (soundBuffer) { - const context = this.runtime.audioEngine && this.runtime.audioEngine.audioContext; + const engine = this.runtime.audioEngine; - if (!context) { + if (!engine) { return Promise.reject(new Error('No Audio Context Detected')); } // Check for newer promise-based API - if (context.decodeAudioData.length === 1) { - return context.decodeAudioData(soundBuffer); - } else { // eslint-disable-line no-else-return - // Fall back to callback API - return new Promise((resolve, reject) => - context.decodeAudioData(soundBuffer, - buffer => resolve(buffer), - error => reject(error) - ) - ); - } + return engine.decodeSoundPlayer({data: {buffer: soundBuffer}}); } /** @@ -778,26 +776,34 @@ class Scratch3MusicBlocks { */ _playDrumNum (util, drumNum) { if (util.runtime.audioEngine === null) return; - if (util.target.audioPlayer === null) return; + if (util.target.sprite.soundBank === null) return; // If we're playing too many sounds, do not play the drum sound. if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) { return; } - const outputNode = util.target.audioPlayer.getInputNode(); - const context = util.runtime.audioEngine.audioContext; - const bufferSource = context.createBufferSource(); - bufferSource.buffer = this._drumBuffers[drumNum]; - bufferSource.connect(outputNode); - bufferSource.start(); - const bufferSourceIndex = this._bufferSources.length; - this._bufferSources.push(bufferSource); + const player = this._drumPlayers[drumNum]; + + if (typeof player === 'undefined') return; + + if (player.isPlaying) { + // Take the internal player state and create a new player with it. + // `.play` does this internally but then instructs the sound to + // stop. + player.take(); + } + + const engine = util.runtime.audioEngine; + const chain = engine.createEffectChain(); + chain.setEffectsFromTarget(util.target); + player.connect(chain); this._concurrencyCounter++; - bufferSource.onended = () => { + player.once('stop', () => { this._concurrencyCounter--; - delete this._bufferSources[bufferSourceIndex]; - }; + }); + + player.play(); } /** @@ -856,7 +862,7 @@ class Scratch3MusicBlocks { */ _playNote (util, note, durationSec) { if (util.runtime.audioEngine === null) return; - if (util.target.audioPlayer === null) return; + if (util.target.sprite.soundBank === null) return; // If we're playing too many sounds, do not play the note. if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) { @@ -871,28 +877,37 @@ class Scratch3MusicBlocks { const sampleIndex = this._selectSampleIndexForNote(note, sampleArray); // If the audio sample has not loaded yet, bail out - if (typeof this._instrumentBufferArrays[inst] === 'undefined') return; - if (typeof this._instrumentBufferArrays[inst][sampleIndex] === 'undefined') return; + if (typeof this._instrumentPlayerArrays[inst] === 'undefined') return; + if (typeof this._instrumentPlayerArrays[inst][sampleIndex] === 'undefined') return; - // Create the audio buffer to play the note, and set its pitch - const context = util.runtime.audioEngine.audioContext; - const bufferSource = context.createBufferSource(); + // Fetch the sound player to play the note. + const engine = util.runtime.audioEngine; - const bufferSourceIndex = this._bufferSources.length; - this._bufferSources.push(bufferSource); + if (!this._instrumentPlayerNoteArrays[inst][note]) { + this._instrumentPlayerNoteArrays[inst][note] = this._instrumentPlayerArrays[inst][sampleIndex].take(); + } - bufferSource.buffer = this._instrumentBufferArrays[inst][sampleIndex]; + const player = this._instrumentPlayerNoteArrays[inst][note]; + + if (player.isPlaying) { + // Take the internal player state and create a new player with it. + // `.play` does this internally but then instructs the sound to + // stop. + player.take(); + } + + const chain = engine.createEffectChain(); + chain.setEffectsFromTarget(util.target); + + // Set its pitch. const sampleNote = sampleArray[sampleIndex]; - bufferSource.playbackRate.value = this._ratioForPitchInterval(note - sampleNote); + const notePitchInterval = this._ratioForPitchInterval(note - sampleNote); - // Create a gain node for this note, and connect it to the sprite's audioPlayer. - const gainNode = context.createGain(); - bufferSource.connect(gainNode); - const outputNode = util.target.audioPlayer.getInputNode(); - gainNode.connect(outputNode); - - // Start playing the note - bufferSource.start(); + // Create a gain node for this note, and connect it to the sprite's + // simulated effectChain. + const context = engine.audioContext; + const releaseGain = context.createGain(); + releaseGain.connect(chain.getInputNode()); // Schedule the release of the note, ramping its gain down to zero, // and then stopping the sound. @@ -902,16 +917,24 @@ class Scratch3MusicBlocks { } const releaseStart = context.currentTime + durationSec; const releaseEnd = releaseStart + releaseDuration; - gainNode.gain.setValueAtTime(1, releaseStart); - gainNode.gain.linearRampToValueAtTime(0.0001, releaseEnd); - bufferSource.stop(releaseEnd); + releaseGain.gain.setValueAtTime(1, releaseStart); + releaseGain.gain.linearRampToValueAtTime(0.0001, releaseEnd); - // Update the concurrency counter this._concurrencyCounter++; - bufferSource.onended = () => { + player.once('stop', () => { this._concurrencyCounter--; - delete this._bufferSources[bufferSourceIndex]; - }; + }); + + // Start playing the note + player.play(); + // Connect the player to the gain node. + player.connect({getInputNode () { + return releaseGain; + }}); + // Set playback now after play creates the outputNode. + player.outputNode.playbackRate.value = notePitchInterval; + // Schedule playback to stop. + player.outputNode.stop(releaseEnd); } /** diff --git a/src/import/load-sound.js b/src/import/load-sound.js index e13db14d2..8f4f7a1a6 100644 --- a/src/import/load-sound.js +++ b/src/import/load-sound.js @@ -8,27 +8,32 @@ const log = require('../util/log'); * @property {Buffer} data - sound data will be written here once loaded. * @param {!Asset} soundAsset - the asset loaded from storage. * @param {!Runtime} runtime - Scratch runtime, used to access the storage module. + * @param {Sprite} sprite - Scratch sprite to add sounds to. * @returns {!Promise} - a promise which will resolve to the sound when ready. */ -const loadSoundFromAsset = function (sound, soundAsset, runtime) { +const loadSoundFromAsset = function (sound, soundAsset, runtime, sprite) { sound.assetId = soundAsset.assetId; if (!runtime.audioEngine) { log.error('No audio engine present; cannot load sound asset: ', sound.md5); return Promise.resolve(sound); } - return runtime.audioEngine.decodeSound(Object.assign( + return runtime.audioEngine.decodeSoundPlayer(Object.assign( {}, sound, {data: soundAsset.data} - )).then(soundId => { - sound.soundId = soundId; + )).then(soundPlayer => { + sound.soundId = soundPlayer.id; // Set the sound sample rate and sample count based on the // the audio buffer from the audio engine since the sound // gets resampled by the audio engine - const soundBuffer = runtime.audioEngine.getSoundBuffer(soundId); + const soundBuffer = soundPlayer.buffer; sound.rate = soundBuffer.sampleRate; sound.sampleCount = soundBuffer.length; + if (sprite.soundBank !== null) { + sprite.soundBank.addSoundPlayer(soundPlayer); + } + return sound; }); }; @@ -39,9 +44,10 @@ const loadSoundFromAsset = function (sound, soundAsset, runtime) { * @property {string} md5 - the MD5 and extension of the sound to be loaded. * @property {Buffer} data - sound data will be written here once loaded. * @param {!Runtime} runtime - Scratch runtime, used to access the storage module. + * @param {Sprite} sprite - Scratch sprite to add sounds to. * @returns {!Promise} - a promise which will resolve to the sound when ready. */ -const loadSound = function (sound, runtime) { +const loadSound = function (sound, runtime, sprite) { if (!runtime.storage) { log.error('No storage module present; cannot load sound asset: ', sound.md5); return Promise.resolve(sound); @@ -52,7 +58,7 @@ const loadSound = function (sound, runtime) { return runtime.storage.load(runtime.storage.AssetType.Sound, md5, ext) .then(soundAsset => { sound.dataFormat = ext; - return loadSoundFromAsset(sound, soundAsset, runtime); + return loadSoundFromAsset(sound, soundAsset, runtime, sprite); }); }; diff --git a/src/serialization/sb2.js b/src/serialization/sb2.js index 9ca378d90..ffde8220a 100644 --- a/src/serialization/sb2.js +++ b/src/serialization/sb2.js @@ -420,7 +420,7 @@ const parseScratchObject = function (object, runtime, extensions, topLevel, zip) // followed by the file ext const assetFileName = `${soundSource.soundID}.${ext}`; soundPromises.push(deserializeSound(sound, runtime, zip, assetFileName) - .then(() => loadSound(sound, runtime))); + .then(() => loadSound(sound, runtime, sprite))); } } diff --git a/src/serialization/sb3.js b/src/serialization/sb3.js index 4f129c171..0ab76a15d 100644 --- a/src/serialization/sb3.js +++ b/src/serialization/sb3.js @@ -800,7 +800,7 @@ const parseScratchObject = function (object, runtime, extensions, zip) { // any translation that needs to happen will happen in the process // of building up the costume object into an sb3 format return deserializeSound(sound, runtime, zip) - .then(() => loadSound(sound, runtime)); + .then(() => loadSound(sound, runtime, sprite)); // Only attempt to load the sound after the deserialization // process has been completed. }); diff --git a/src/sprites/rendered-target.js b/src/sprites/rendered-target.js index aac0b059a..93e7d8c7f 100644 --- a/src/sprites/rendered-target.js +++ b/src/sprites/rendered-target.js @@ -170,21 +170,30 @@ class RenderedTarget extends Target { } } + get audioPlayer () { + /* eslint-disable no-console */ + console.warn('get audioPlayer deprecated, please update to use .sprite.soundBank methods'); + console.warn(new Error('stack for debug').stack); + /* eslint-enable no-console */ + const bank = this.sprite.soundBank; + const audioPlayerProxy = { + playSound: soundId => bank.play(this, soundId) + }; + + Object.defineProperty(this, 'audioPlayer', { + configurable: false, + enumerable: true, + writable: false, + value: audioPlayerProxy + }); + + return audioPlayerProxy; + } + /** * Initialize the audio player for this sprite or clone. */ initAudio () { - this.audioPlayer = null; - if (this.runtime && this.runtime.audioEngine) { - this.audioPlayer = this.runtime.audioEngine.createPlayer(); - // If this is a clone, it gets a reference to its parent's activeSoundPlayers object. - if (!this.isOriginal) { - const parent = this.sprite.clones[0]; - if (parent && parent.audioPlayer) { - this.audioPlayer.activeSoundPlayers = parent.audioPlayer.activeSoundPlayers; - } - } - } } /** @@ -1034,9 +1043,8 @@ class RenderedTarget extends Target { */ onStopAll () { this.clearEffects(); - if (this.audioPlayer) { - this.audioPlayer.stopAllSounds(); - this.audioPlayer.clearEffects(); + if (this.sprite.soundBank) { + this.sprite.soundBank.stopAllSounds(this); } } @@ -1122,6 +1130,9 @@ class RenderedTarget extends Target { dispose () { this.runtime.changeCloneCounter(-1); this.runtime.stopForTarget(this); + if (this.sprite.soundBank) { + this.sprite.soundBank.stopAllSounds(this); + } this.sprite.removeClone(this); if (this.renderer && this.drawableID !== null) { this.renderer.destroyDrawable(this.drawableID, this.isStage ? @@ -1132,10 +1143,6 @@ class RenderedTarget extends Target { this.runtime.requestRedraw(); } } - if (this.audioPlayer) { - this.audioPlayer.stopAllSounds(); - this.audioPlayer.dispose(); - } } } diff --git a/src/sprites/sprite.js b/src/sprites/sprite.js index de735ecc3..bc61960a0 100644 --- a/src/sprites/sprite.js +++ b/src/sprites/sprite.js @@ -8,7 +8,8 @@ const StageLayering = require('../engine/stage-layering'); class Sprite { /** * Sprite to be used on the Scratch stage. - * All clones of a sprite have shared blocks, shared costumes, shared variables. + * All clones of a sprite have shared blocks, shared costumes, shared variables, + * shared sounds, etc. * @param {?Blocks} blocks Shared blocks object for all clones of sprite. * @param {Runtime} runtime Reference to the runtime. * @constructor @@ -47,6 +48,11 @@ class Sprite { * @type {Array.} */ this.clones = []; + + this.soundBank = null; + if (this.runtime && this.runtime.audioEngine) { + this.soundBank = this.runtime.audioEngine.createBank(); + } } /** @@ -149,12 +155,18 @@ class Sprite { newSprite.sounds = this.sounds.map(sound => { const newSound = Object.assign({}, sound); const soundAsset = this.runtime.storage.get(sound.assetId); - assetPromises.push(loadSoundFromAsset(newSound, soundAsset, this.runtime)); + assetPromises.push(loadSoundFromAsset(newSound, soundAsset, this.runtime, this)); return newSound; }); return Promise.all(assetPromises).then(() => newSprite); } + + dispose () { + if (this.soundBank) { + this.soundBank.dispose(); + } + } } module.exports = Sprite; diff --git a/src/virtual-machine.js b/src/virtual-machine.js index 767641eec..57ae97e76 100644 --- a/src/virtual-machine.js +++ b/src/virtual-machine.js @@ -495,7 +495,7 @@ class VirtualMachine extends EventEmitter { duplicateSound (soundIndex) { const originalSound = this.editingTarget.getSounds()[soundIndex]; const clone = Object.assign({}, originalSound); - return loadSound(clone, this.runtime).then(() => { + return loadSound(clone, this.runtime, this.editingTarget.sprite).then(() => { this.editingTarget.addSound(clone, soundIndex + 1); this.emitTargetsUpdate(); }); @@ -525,7 +525,7 @@ class VirtualMachine extends EventEmitter { * @returns {?Promise} - a promise that resolves when the sound has been decoded and added */ addSound (soundObject) { - return loadSound(soundObject, this.runtime).then(() => { + return loadSound(soundObject, this.runtime, this.editingTarget.sprite).then(() => { this.editingTarget.addSound(soundObject); this.emitTargetsUpdate(); }); @@ -549,7 +549,7 @@ class VirtualMachine extends EventEmitter { getSoundBuffer (soundIndex) { const id = this.editingTarget.sprite.sounds[soundIndex].soundId; if (id && this.runtime && this.runtime.audioEngine) { - return this.runtime.audioEngine.getSoundBuffer(id); + return this.editingTarget.sprite.soundBank.getSoundPlayer(id).buffer; } return null; } @@ -564,7 +564,7 @@ class VirtualMachine extends EventEmitter { const sound = this.editingTarget.sprite.sounds[soundIndex]; const id = sound ? sound.soundId : null; if (id && this.runtime && this.runtime.audioEngine) { - this.runtime.audioEngine.updateSoundBuffer(id, newBuffer); + this.editingTarget.sprite.soundBank.getSoundPlayer(id).buffer = newBuffer; } // Update sound in runtime if (soundEncoding) { @@ -966,8 +966,8 @@ class VirtualMachine extends EventEmitter { shareSoundToTarget (soundIndex, targetId) { const originalSound = this.editingTarget.getSounds()[soundIndex]; const clone = Object.assign({}, originalSound); - return loadSound(clone, this.runtime).then(() => { - const target = this.runtime.getTargetById(targetId); + const target = this.runtime.getTargetById(targetId); + return loadSound(clone, this.runtime, target.sprite).then(() => { if (target) { target.addSound(clone); this.emitTargetsUpdate(); diff --git a/test/unit/blocks_sounds.js b/test/unit/blocks_sounds.js index 9a44e7f9f..53acc8aff 100644 --- a/test/unit/blocks_sounds.js +++ b/test/unit/blocks_sounds.js @@ -11,10 +11,10 @@ const util = { {name: 'second name', soundId: 'second soundId'}, {name: 'third name', soundId: 'third soundId'}, {name: '6', soundId: 'fourth soundId'} - ] - }, - audioPlayer: { - playSound: soundId => (playedSound = soundId) + ], + soundBank: { + playSound: (target, soundId) => (playedSound = soundId) + } } } };