From 0059ff53e1f7cca3be06a545f04f63985285c153 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 11 Jun 2018 15:20:48 -0400 Subject: [PATCH 1/8] start audio param transition at now Explicitly start the param transitions at the audio context's now time. --- src/effects/PanEffect.js | 12 ++++++++++-- src/effects/VolumeEffect.js | 6 +++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/effects/PanEffect.js b/src/effects/PanEffect.js index ea50c2f..d58558a 100644 --- a/src/effects/PanEffect.js +++ b/src/effects/PanEffect.js @@ -58,8 +58,16 @@ class PanEffect extends Effect { const leftVal = Math.cos(p * Math.PI / 2); const rightVal = Math.sin(p * Math.PI / 2); - this.leftGain.gain.setTargetAtTime(leftVal, 0, this.audioEngine.DECAY_TIME); - this.rightGain.gain.setTargetAtTime(rightVal, 0, this.audioEngine.DECAY_TIME); + this.leftGain.gain.setTargetAtTime( + leftVal, + this.audioEngine.audioContext.currentTime, + this.audioEngine.DECAY_TIME + ); + this.rightGain.gain.setTargetAtTime( + rightVal, + this.audioEngine.audioContext.currentTime, + this.audioEngine.DECAY_TIME + ); } /** diff --git a/src/effects/VolumeEffect.js b/src/effects/VolumeEffect.js index 8e536d7..32204f0 100644 --- a/src/effects/VolumeEffect.js +++ b/src/effects/VolumeEffect.js @@ -34,7 +34,11 @@ class VolumeEffect extends Effect { this.value = value; // A gain of 1 is normal. Scale down scratch's volume value. Apply the // change over a tiny period of time. - this.outputNode.gain.setTargetAtTime(value / 100, 0, this.audioEngine.DECAY_TIME); + this.outputNode.gain.setTargetAtTime( + value / 100, + this.audioEngine.audioContext.currentTime, + this.audioEngine.DECAY_TIME + ); } /** From 87f400b27d68d622746dd6bb0c7801873feaa35a Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 11 Jun 2018 15:23:12 -0400 Subject: [PATCH 2/8] set DECAY_TIME to 25ms Use a DECAY_TIME that envelopes half of the lowest frequency a person can hear. --- src/AudioEngine.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/AudioEngine.js b/src/AudioEngine.js index f76d150..23cf144 100644 --- a/src/AudioEngine.js +++ b/src/AudioEngine.js @@ -77,13 +77,20 @@ class AudioEngine { } /** - * A short duration, for use as a time constant for exponential audio parameter transitions. - * See: - * https://developer.mozilla.org/en-US/docs/Web/API/AudioParam/setTargetAtTime + * A short duration to transition audio prarameters. + * + * Used as a time constant for exponential transitions. A general value + * must be large enough that it does not cute off lower frequency, or bass, + * sounds. Human hearing lower limit is ~20Hz making a safe value 25 + * milliseconds or 0.025 seconds, where half of a 20Hz wave will play along + * with the DECAY. Higher frequencies will play multiple waves during the + * same amount of time and avoid clipping. + * + * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/AudioParam/setTargetAtTime} * @const {number} */ get DECAY_TIME () { - return 0.001; + return 0.025; } /** From 16883f60f8f85297a501b237d2efbd3b3a261e07 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 11 Jun 2018 15:23:58 -0400 Subject: [PATCH 3/8] support null AudioPlayer in Effect Support null AudioPlayer so Effect chains can be declared without it. --- src/effects/Effect.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/effects/Effect.js b/src/effects/Effect.js index 1dc1bbe..8a4da12 100644 --- a/src/effects/Effect.js +++ b/src/effects/Effect.js @@ -38,7 +38,7 @@ class Effect { * @return {boolean} is the effect affecting the graph? */ get _isPatch () { - return this.initialized && this.value !== this.DEFAULT_VALUE; + return this.initialized && (this.value !== this.DEFAULT_VALUE || this.audioPlayer === null); } /** @@ -138,7 +138,9 @@ class Effect { } if (this.lastEffect === null) { - this.audioPlayer.connect(this); + if (this.audioPlayer !== null) { + this.audioPlayer.connect(this); + } } else { this.lastEffect.connect(this); } From 06a9af4dc37c263298df79457f4a75e1aed87da5 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 11 Jun 2018 15:24:41 -0400 Subject: [PATCH 4/8] implement new SoundPlayer - Add GreenPlayer as SoundPlayer replacement - Standalone playback separate from an AudioPlayer - Each GreenPlayer has their own VolumeEffect for fading out - play() while already isPlaying fades out the last copy - stop() fades the sound out - stopImmediate() hard stops the sound --- src/GreenPlayer.js | 166 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 src/GreenPlayer.js diff --git a/src/GreenPlayer.js b/src/GreenPlayer.js new file mode 100644 index 0000000..76eda18 --- /dev/null +++ b/src/GreenPlayer.js @@ -0,0 +1,166 @@ +const {EventEmitter} = require('events'); + +const VolumeEffect = require('./effects/VolumeEffect'); + +class SoundPlayer extends EventEmitter { + constructor (audioEngine, {id, buffer}) { + super(); + + this.id = id; + + this.audioEngine = audioEngine; + this.buffer = buffer; + + this.outputNode = null; + this.target = null; + + this.initialized = false; + this.isPlaying = false; + this.playbackRate = 1; + + this.onEnd = this.onEnd.bind(this); + } + + onEnd () { + this.emit('stop'); + } + + initialize () { + this.outputNode = this.audioEngine.audioContext.createBufferSource(); + this.outputNode.playbackRate.value = this.playbackRate; + this.outputNode.buffer = this.buffer; + + this.outputNode.addEventListener('end', this.onEnd); + + this.volumeEffect = new VolumeEffect(this.audioEngine, this, null); + + this.initialized = true; + + if (this.target !== null) { + this.connect(this.target); + this.setPlaybackRate(this.playbackRate); + } + } + + connect (target) { + if (target === this.volumeEffect) { + this.outputNode.disconnect(); + this.outputNode.connect(this.volumeEffect.getInputNode()); + return; + } + + this.target = target; + + if (!this.initialized) { + return; + } + + this.volumeEffect.connect(target); + + return this; + } + + dispose () { + if (!this.initialized) { + return; + } + + this.stopImmediately(); + + this.volumeEffect.dispose(); + + this.outputNode.disconnect(); + this.outputNode = null; + + this.target = null; + + this.initialized = false; + } + + take () { + if (this.outputNode) { + this.outputNode.removeEventListener('end', this.onEnd); + } + + const taken = new SoundPlayer(this.audioEngine, this); + taken.outputNode = this.outputNode; + if (this.volumeEffect !== null) { + taken.volumeEffect.set(this.volumeEffect.value); + } + if (this.target !== null) { + taken.connect(this.target); + } + taken.initialized = this.initialized; + taken.isPlaying = this.isPlaying; + taken.playbackRate = this.playbackRate; + + if (this.isPlaying) { + this.emit('stop'); + taken.emit('play'); + } + + this.outputNode = null; + if (this.volumeEffect !== null) { + this.volumeEffect.dispose(); + } + this.volumeEffect = null; + this.target = null; + this.initialized = false; + this.isPlaying = false; + } + + play () { + if (this.isPlaying) { + // Spawn a Player with the current buffer source, and play for a + // short period until its volume is 0 and release it to be + // eventually garbage collected. + this.take().stop(); + } + + if (!this.initialized) { + this.initialize(); + } + + this.volumeEffect.set(this.volumeEffect.DEFAULT_VALUE); + this.outputNode.start(); + + this.isPlaying = true; + + this.emit('play'); + } + + stop () { + if (!this.isPlaying) { + return; + } + + this.volumeEffect.set(0); + this.outputNode.stop(this.audioEngine.audioEngineoContext.currentTime + this.audioEngine.DECAY_TIME); + + this.isPlaying = false; + + this.emit('stop'); + } + + stopImmediately () { + if (!this.isPlaying) { + return; + } + + this.outputNode.stop(); + + this.isPlaying = false; + + this.emit('stop'); + } + + setPlaybackRate (value) { + this.playbackRate = value; + + if (this.initialized) { + this.outputNode.playbackRate.value = value; + } + } +} + +module.exports = SoundPlayer; From a5702a7d49956f8c0dde3404de27f563ba54c103 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 11 Jun 2018 15:26:45 -0400 Subject: [PATCH 5/8] add decodeSoundPlayer to AudioEngine - decodeSoundPlayer returns a SoundPlayer instance instance of the sound id - Write deprecation notes in AudioEngine about audioBuffers --- src/AudioEngine.js | 90 +++++++++++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 24 deletions(-) diff --git a/src/AudioEngine.js b/src/AudioEngine.js index 23cf144..8c7e142 100644 --- a/src/AudioEngine.js +++ b/src/AudioEngine.js @@ -5,9 +5,9 @@ const log = require('./log'); const uid = require('./uid'); const ADPCMSoundDecoder = require('./ADPCMSoundDecoder'); - const AudioPlayer = require('./AudioPlayer'); const Loudness = require('./Loudness'); +const SoundPlayer = require('./GreenPlayer'); /** * Wrapper to ensure that audioContext.decodeAudioData is a promise @@ -103,40 +103,77 @@ class AudioEngine { /** * Decode a sound, decompressing it into audio samples. - * Store a reference to it the sound in the audioBuffers dictionary, indexed by soundId - * @param {object} sound - an object containing audio data and metadata for a sound - * @property {Buffer} data - sound data loaded from scratch-storage. - * @returns {?Promise} - a promise which will resolve to the soundId if decoded and stored. + * @param {object} sound - an object containing audio data and metadata for + * a sound + * @param {Buffer} sound.data - sound data loaded from scratch-storage + * @returns {?Promise} - a promise which will resolve to the sound id and + * buffer if decoded */ - decodeSound (sound) { - // Make a copy of the buffer because decoding detaches the original buffer + _decodeSound (sound) { + // Make a copy of the buffer because decoding detaches the original + // buffer const bufferCopy1 = sound.data.buffer.slice(0); + // todo: multiple decodings of the same buffer create duplicate decoded + // copies in audioBuffers. Create a hash id of the buffer or deprecate + // audioBuffers to avoid memory issues for large audio buffers. const soundId = uid(); - // Partially apply updateSoundBuffer function with the current - // soundId so that it's ready to be used on successfully decoded audio - const addDecodedAudio = this.updateSoundBuffer.bind(this, soundId); - // Attempt to decode the sound using the browser's native audio data decoder - // If that fails, attempt to decode as ADPCM - return decodeAudioData(this.audioContext, bufferCopy1).then( - addDecodedAudio, - () => { + // Attempt to decode the sound using the browser's native audio data + // decoder If that fails, attempt to decode as ADPCM + const decoding = decodeAudioData(this.audioContext, bufferCopy1) + .catch(() => { // The audio context failed to parse the sound data // we gave it, so try to decode as 'adpcm' // First we need to create another copy of our original data const bufferCopy2 = sound.data.buffer.slice(0); // Try decoding as adpcm - return (new ADPCMSoundDecoder(this.audioContext)).decode(bufferCopy2) - .then( - addDecodedAudio, - error => { - log.warn('audio data could not be decoded', error); - } - ); - } - ); + return new ADPCMSoundDecoder(this.audioContext).decode(bufferCopy2); + }) + .then( + buffer => ([soundId, buffer]), + error => { + log.warn('audio data could not be decoded', error); + } + ); + + return decoding; + } + + /** + * Decode a sound, decompressing it into audio samples. + * + * Store a reference to it the sound in the audioBuffers dictionary, + * indexed by soundId. + * + * @param {object} sound - an object containing audio data and metadata for + * a sound + * @param {Buffer} sound.data - sound data loaded from scratch-storage + * @returns {?Promise} - a promise which will resolve to the sound id + */ + decodeSound (sound) { + return this._decodeSound(sound) + .then(([id, buffer]) => { + this.audioBuffers[id] = buffer; + return id; + }); + } + + /** + * Decode a sound, decompressing it into audio samples. + * + * Create a SoundPlayer instance that can be used to play the sound and + * stop and fade out playback. + * + * @param {object} sound - an object containing audio data and metadata for + * a sound + * @param {Buffer} sound.data - sound data loaded from scratch-storage + * @returns {?Promise} - a promise which will resolve to the buffer + */ + decodeSoundPlayer (sound) { + return this._decodeSound(sound) + .then(([id, buffer]) => new SoundPlayer(this, {id, buffer})); } /** @@ -145,6 +182,11 @@ class AudioEngine { * @return {AudioBuffer} the buffer corresponding to the given sound id. */ getSoundBuffer (soundId) { + // todo: Deprecate audioBuffers. If something wants to hold onto the + // buffer, it should. Otherwise buffers need to be able to release their + // decoded memory to avoid running out of memory which is possible with + // enough large audio buffers as they are full 16bit pcm waveforms for + // each audio channel. return this.audioBuffers[soundId]; } From 5c822e654205a9b0a54106e419a11c611a160f42 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Tue, 12 Jun 2018 09:18:48 -0400 Subject: [PATCH 6/8] fix: listen to ended event to note playback stopping - Fix playing a sound a second time once the first playback finished --- src/GreenPlayer.js | 51 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/src/GreenPlayer.js b/src/GreenPlayer.js index 76eda18..1ea9a26 100644 --- a/src/GreenPlayer.js +++ b/src/GreenPlayer.js @@ -2,6 +2,8 @@ const {EventEmitter} = require('events'); const VolumeEffect = require('./effects/VolumeEffect'); +const ON_ENDED = 'ended'; + class SoundPlayer extends EventEmitter { constructor (audioEngine, {id, buffer}) { super(); @@ -17,31 +19,48 @@ class SoundPlayer extends EventEmitter { this.initialized = false; this.isPlaying = false; this.playbackRate = 1; - - this.onEnd = this.onEnd.bind(this); } - onEnd () { + /** + * Handle any event we have told the output node to listen for. + */ + handleEvent (event) { + if (event.type === ON_ENDED) { + this.onEnded(); + } + } + + onEnded () { this.emit('stop'); + + this.isPlaying = false; } - initialize () { + _createSource () { + if (this.outputNode !== null) { + this.outputNode.removeEventListener(ON_ENDED, this); + this.outputNode.disconnect(); + } + this.outputNode = this.audioEngine.audioContext.createBufferSource(); this.outputNode.playbackRate.value = this.playbackRate; this.outputNode.buffer = this.buffer; - this.outputNode.addEventListener('end', this.onEnd); - - this.volumeEffect = new VolumeEffect(this.audioEngine, this, null); - - this.initialized = true; + this.outputNode.addEventListener(ON_ENDED, this); if (this.target !== null) { this.connect(this.target); - this.setPlaybackRate(this.playbackRate); } } + initialize () { + this.initialized = true; + + this.volumeEffect = new VolumeEffect(this.audioEngine, this, null); + + this._createSource(); + } + connect (target) { if (target === this.volumeEffect) { this.outputNode.disconnect(); @@ -79,7 +98,7 @@ class SoundPlayer extends EventEmitter { take () { if (this.outputNode) { - this.outputNode.removeEventListener('end', this.onEnd); + this.outputNode.removeEventListener(ON_ENDED, this); } const taken = new SoundPlayer(this.audioEngine, this); @@ -119,6 +138,8 @@ class SoundPlayer extends EventEmitter { if (!this.initialized) { this.initialize(); + } else { + this._createSource(); } this.volumeEffect.set(this.volumeEffect.DEFAULT_VALUE); @@ -135,7 +156,7 @@ class SoundPlayer extends EventEmitter { } this.volumeEffect.set(0); - this.outputNode.stop(this.audioEngine.audioEngineoContext.currentTime + this.audioEngine.DECAY_TIME); + this.outputNode.stop(this.audioEngine.audioContext.currentTime + this.audioEngine.DECAY_TIME); this.isPlaying = false; @@ -154,6 +175,12 @@ class SoundPlayer extends EventEmitter { this.emit('stop'); } + finished () { + return new Promise(resolve => { + this.once('stop', resolve); + }); + } + setPlaybackRate (value) { this.playbackRate = value; From 90589b861d46c14d1ae7c365e875f3dd893739a6 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Tue, 12 Jun 2018 09:20:21 -0400 Subject: [PATCH 7/8] fix: fix taking a SoundPlayer's state --- src/GreenPlayer.js | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/GreenPlayer.js b/src/GreenPlayer.js index 1ea9a26..e6d0a41 100644 --- a/src/GreenPlayer.js +++ b/src/GreenPlayer.js @@ -102,16 +102,18 @@ class SoundPlayer extends EventEmitter { } const taken = new SoundPlayer(this.audioEngine, this); - taken.outputNode = this.outputNode; - if (this.volumeEffect !== null) { - taken.volumeEffect.set(this.volumeEffect.value); - } - if (this.target !== null) { - taken.connect(this.target); - } - taken.initialized = this.initialized; - taken.isPlaying = this.isPlaying; taken.playbackRate = this.playbackRate; + if (this.isPlaying) { + taken.isPlaying = this.isPlaying; + taken.initialize(); + taken.outputNode.disconnect(); + taken.outputNode = this.outputNode; + taken.outputNode.addEventListener(ON_ENDED, taken); + taken.volumeEffect.set(this.volumeEffect.value); + if (this.target !== null) { + taken.connect(this.target); + } + } if (this.isPlaying) { this.emit('stop'); @@ -123,9 +125,10 @@ class SoundPlayer extends EventEmitter { this.volumeEffect.dispose(); } this.volumeEffect = null; - this.target = null; this.initialized = false; this.isPlaying = false; + + return taken; } play () { From 46b7c6d37bf801212c1cea76a6733de5b0f37c9f Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Tue, 12 Jun 2018 09:40:59 -0400 Subject: [PATCH 8/8] document GreenPlayer --- src/GreenPlayer.js | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/GreenPlayer.js b/src/GreenPlayer.js index e6d0a41..fb70e2e 100644 --- a/src/GreenPlayer.js +++ b/src/GreenPlayer.js @@ -2,9 +2,22 @@ const {EventEmitter} = require('events'); const VolumeEffect = require('./effects/VolumeEffect'); +/** + * Name of event that indicates playback has ended. + * @const {string} + */ const ON_ENDED = 'ended'; class SoundPlayer extends EventEmitter { + /** + * Play sounds that stop without audible clipping. + * + * @param {AudioEngine} audioEngine - engine to play sounds on + * @param {object} data - required data for sound playback + * @param {string} data.id - a unique id for this sound + * @param {ArrayBuffer} data.buffer - buffer of the sound's waveform to play + * @constructor + */ constructor (audioEngine, {id, buffer}) { super(); @@ -23,6 +36,7 @@ class SoundPlayer extends EventEmitter { /** * Handle any event we have told the output node to listen for. + * @param {Event} event - dom event to handle */ handleEvent (event) { if (event.type === ON_ENDED) { @@ -30,12 +44,19 @@ class SoundPlayer extends EventEmitter { } } + /** + * Event listener for when playback ends. + */ onEnded () { this.emit('stop'); this.isPlaying = false; } + /** + * Create the buffer source node during initialization or secondary + * playback. + */ _createSource () { if (this.outputNode !== null) { this.outputNode.removeEventListener(ON_ENDED, this); @@ -53,6 +74,9 @@ class SoundPlayer extends EventEmitter { } } + /** + * Initialize the player for first playback. + */ initialize () { this.initialized = true; @@ -61,6 +85,11 @@ class SoundPlayer extends EventEmitter { this._createSource(); } + /** + * Connect the player to the engine or an effect chain. + * @param {object} target - object to connect to + * @returns {object} - return this sound player + */ connect (target) { if (target === this.volumeEffect) { this.outputNode.disconnect(); @@ -79,6 +108,9 @@ class SoundPlayer extends EventEmitter { return this; } + /** + * Teardown the player. + */ dispose () { if (!this.initialized) { return; @@ -87,6 +119,7 @@ class SoundPlayer extends EventEmitter { this.stopImmediately(); this.volumeEffect.dispose(); + this.volumeEffect = null; this.outputNode.disconnect(); this.outputNode = null; @@ -96,6 +129,15 @@ class SoundPlayer extends EventEmitter { this.initialized = false; } + /** + * Take the internal state of this player and create a new player from + * that. Restore the state of this player to that before its first playback. + * + * The returned player can be used to stop the original playback or + * continue it without manipulation from the original player. + * + * @returns {SoundPlayer} - new SoundPlayer with old state + */ take () { if (this.outputNode) { this.outputNode.removeEventListener(ON_ENDED, this); @@ -131,6 +173,12 @@ class SoundPlayer extends EventEmitter { return taken; } + /** + * Start playback for this sound. + * + * If the sound is already playing it will stop playback with a quick fade + * out. + */ play () { if (this.isPlaying) { // Spawn a Player with the current buffer source, and play for a @@ -153,6 +201,9 @@ class SoundPlayer extends EventEmitter { this.emit('play'); } + /** + * Stop playback after quickly fading out. + */ stop () { if (!this.isPlaying) { return; @@ -166,6 +217,9 @@ class SoundPlayer extends EventEmitter { this.emit('stop'); } + /** + * Stop immediately without fading out. May cause audible clipping. + */ stopImmediately () { if (!this.isPlaying) { return; @@ -178,12 +232,20 @@ class SoundPlayer extends EventEmitter { this.emit('stop'); } + /** + * Return a promise that resolves when the sound next finishes. + * @returns {Promise} - resolves when the sound finishes + */ finished () { return new Promise(resolve => { this.once('stop', resolve); }); } + /** + * Set the sound's playback rate. + * @param {number} value - playback rate. Default is 1. + */ setPlaybackRate (value) { this.playbackRate = value;