diff --git a/src/AudioEngine.js b/src/AudioEngine.js new file mode 100644 index 0000000..8746980 --- /dev/null +++ b/src/AudioEngine.js @@ -0,0 +1,181 @@ +const StartAudioContext = require('startaudiocontext'); +const AudioContext = require('audio-context'); + +const log = require('./log'); +const uid = require('./uid'); + +const ADPCMSoundDecoder = require('./ADPCMSoundDecoder'); + +const AudioPlayer = require('./AudioPlayer'); +const Loudness = require('./Loudness'); + +/** + * Wrapper to ensure that audioContext.decodeAudioData is a promise + * @param {object} audioContext The current AudioContext + * @param {ArrayBuffer} buffer Audio data buffer to decode + * @return {Promise} A promise that resolves to the decoded audio + */ +const decodeAudioData = function (audioContext, buffer) { + // Check for newer promise-based API + if (audioContext.decodeAudioData.length === 1) { + return audioContext.decodeAudioData(buffer); + } + // Fall back to callback API + return new Promise((resolve, reject) => { + audioContext.decodeAudioData(buffer, + decodedAudio => resolve(decodedAudio), + error => reject(error) + ); + }); +}; + +/** + * There is a single instance of the AudioEngine. It handles global audio + * properties and effects, loads all the audio buffers for sounds belonging to + * sprites. + */ +class AudioEngine { + constructor () { + /** + * AudioContext to play and manipulate sounds with a graph of source + * and effect nodes. + * @type {AudioContext} + */ + this.audioContext = new AudioContext(); + StartAudioContext(this.audioContext); + + /** + * Master GainNode that all sounds plays through. Changing this node + * will change the volume for all sounds. + * @type {GainNode} + */ + this.input = this.audioContext.createGain(); + this.input.connect(this.audioContext.destination); + + /** + * a map of soundIds to audio buffers, holding sounds for all sprites + * @type {Object} + */ + this.audioBuffers = {}; + + /** + * A Loudness detector. + * @type {Loudness} + */ + this.loudness = null; + } + + /** + * Names of the audio effects. + * @enum {string} + */ + get EFFECT_NAMES () { + return { + pitch: 'pitch', + pan: 'pan' + }; + } + + /** + * A short duration, for use as a time constant for exponential audio parameter transitions. + * See: + * https://developer.mozilla.org/en-US/docs/Web/API/AudioParam/setTargetAtTime + * @const {number} + */ + get DECAY_TIME () { + return 0.001; + } + + /** + * Decode a sound, decompressing it into audio samples. + * Store a reference to it the sound in the audioBuffers dictionary, indexed by soundId + * @param {object} sound - an object containing audio data and metadata for a sound + * @property {Buffer} data - sound data loaded from scratch-storage. + * @returns {?Promise} - a promise which will resolve to the soundId if decoded and stored. + */ + decodeSound (sound) { + // Make a copy of the buffer because decoding detaches the original buffer + const bufferCopy1 = sound.data.buffer.slice(0); + + const soundId = uid(); + // Partially apply updateSoundBuffer function with the current + // soundId so that it's ready to be used on successfully decoded audio + const addDecodedAudio = this.updateSoundBuffer.bind(this, soundId); + + // Attempt to decode the sound using the browser's native audio data decoder + // If that fails, attempt to decode as ADPCM + return decodeAudioData(this.audioContext, bufferCopy1).then( + addDecodedAudio, + () => { + // The audio context failed to parse the sound data + // we gave it, so try to decode as 'adpcm' + + // First we need to create another copy of our original data + const bufferCopy2 = sound.data.buffer.slice(0); + // Try decoding as adpcm + return (new ADPCMSoundDecoder(this.audioContext)).decode(bufferCopy2) + .then( + addDecodedAudio, + error => { + log.warn('audio data could not be decoded', error); + } + ); + } + ); + } + + /** + * Retrieve the audio buffer as held in memory for a given sound id. + * @param {!string} soundId - the id of the sound buffer to get + * @return {AudioBuffer} the buffer corresponding to the given sound id. + */ + getSoundBuffer (soundId) { + return this.audioBuffers[soundId]; + } + + /** + * Add or update the in-memory audio buffer to a new one by soundId. + * @param {!string} soundId - the id of the sound buffer to update. + * @param {AudioBuffer} newBuffer - the new buffer to swap in. + * @return {string} The uid of the sound that was updated or added + */ + updateSoundBuffer (soundId, newBuffer) { + this.audioBuffers[soundId] = newBuffer; + return soundId; + } + + /** + * An older version of the AudioEngine had this function to load all sounds + * This is a stub to provide a warning when it is called + * @todo remove this + */ + loadSounds () { + log.warn('The loadSounds function is no longer available. Please use Scratch Storage.'); + } + + /** + * Get the current loudness of sound received by the microphone. + * Sound is measured in RMS and smoothed. + * @return {number} loudness scaled 0 to 100 + */ + getLoudness () { + // The microphone has not been set up, so try to connect to it + if (!this.loudness) { + this.loudness = new Loudness(this.audioContext); + } + + return this.loudness.getLoudness(); + } + + /** + * Create an AudioPlayer. Each sprite or clone has an AudioPlayer. + * It includes a reference to the AudioEngine so it can use global + * functionality such as playing notes. + * @return {AudioPlayer} new AudioPlayer instance + */ + createPlayer () { + return new AudioPlayer(this); + } +} + +module.exports = AudioEngine; diff --git a/src/AudioPlayer.js b/src/AudioPlayer.js new file mode 100644 index 0000000..27d7310 --- /dev/null +++ b/src/AudioPlayer.js @@ -0,0 +1,134 @@ +const PitchEffect = require('./effects/PitchEffect'); +const PanEffect = require('./effects/PanEffect'); + +const SoundPlayer = require('./SoundPlayer'); + +class AudioPlayer { + /** + * Each sprite or clone has an audio player + * the audio player handles sound playback, volume, and the sprite-specific audio effects: + * pitch and pan + * @param {AudioEngine} audioEngine AudioEngine for player + * @constructor + */ + constructor (audioEngine) { + this.audioEngine = audioEngine; + + // Create the audio effects + this.pitchEffect = new PitchEffect(); + this.panEffect = new PanEffect(this.audioEngine); + + // Chain the audio effects together + // effectsNode -> panEffect -> audioEngine.input + this.effectsNode = this.audioEngine.audioContext.createGain(); + this.effectsNode.connect(this.panEffect.input); + this.panEffect.connect(this.audioEngine.input); + + // reset effects to their default parameters + this.clearEffects(); + + // sound players that are currently playing, indexed by the sound's soundId + this.activeSoundPlayers = {}; + } + + /** + * Get this sprite's input node, so that other objects can route sound through it. + * @return {AudioNode} the AudioNode for this sprite's input + */ + getInputNode () { + return this.effectsNode; + } + + /** + * Play a sound + * @param {string} soundId - the soundId id of a sound file + * @return {Promise} a Promise that resolves when the sound finishes playing + */ + playSound (soundId) { + // if this sound is not in the audio engine, return + if (!this.audioEngine.audioBuffers[soundId]) { + return; + } + + // if this sprite or clone is already playing this sound, stop it first + if (this.activeSoundPlayers[soundId]) { + this.activeSoundPlayers[soundId].stop(); + } + + // create a new soundplayer to play the sound + const player = new SoundPlayer(this.audioEngine.audioContext); + player.setBuffer(this.audioEngine.audioBuffers[soundId]); + player.connect(this.effectsNode); + this.pitchEffect.updatePlayer(player); + player.start(); + + // add it to the list of active sound players + this.activeSoundPlayers[soundId] = player; + + // remove sounds that are not playing from the active sound players array + for (const id in this.activeSoundPlayers) { + if (this.activeSoundPlayers.hasOwnProperty(id)) { + if (!this.activeSoundPlayers[id].isPlaying) { + delete this.activeSoundPlayers[id]; + } + } + } + + return player.finished(); + } + + /** + * Stop all sounds that are playing + */ + stopAllSounds () { + // stop all active sound players + for (const soundId in this.activeSoundPlayers) { + this.activeSoundPlayers[soundId].stop(); + } + } + + /** + * Set an audio effect to a value + * @param {string} effect - the name of the effect + * @param {number} value - the value to set the effect to + */ + setEffect (effect, value) { + switch (effect) { + case this.audioEngine.EFFECT_NAMES.pitch: + this.pitchEffect.set(value, this.activeSoundPlayers); + break; + case this.audioEngine.EFFECT_NAMES.pan: + this.panEffect.set(value); + break; + } + } + + /** + * Clear all audio effects + */ + clearEffects () { + this.panEffect.set(0); + this.pitchEffect.set(0, this.activeSoundPlayers); + if (this.audioEngine === null) return; + this.effectsNode.gain.setTargetAtTime(1.0, 0, this.audioEngine.DECAY_TIME); + } + + /** + * Set the volume for sounds played by this AudioPlayer + * @param {number} value - the volume in range 0-100 + */ + setVolume (value) { + if (this.audioEngine === null) return; + this.effectsNode.gain.setTargetAtTime(value / 100, 0, this.audioEngine.DECAY_TIME); + } + + /** + * Clean up and disconnect audio nodes. + */ + dispose () { + this.panEffect.dispose(); + this.effectsNode.disconnect(); + } +} + +module.exports = AudioPlayer; diff --git a/src/Loudness.js b/src/Loudness.js new file mode 100644 index 0000000..e459234 --- /dev/null +++ b/src/Loudness.js @@ -0,0 +1,82 @@ +const log = require('./log'); + +class Loudness { + /** + * Instrument and detect a loudness value from a local microphone. + * @param {AudioContext} audioContext - context to create nodes from for + * detecting loudness + * @constructor + */ + constructor (audioContext) { + /** + * AudioContext the mic will connect to and provide analysis of + * @type {AudioContext} + */ + this.audioContext = audioContext; + + /** + * Are we connecting to the mic yet? + * @type {Boolean} + */ + this.connectingToMic = false; + + /** + * microphone, for measuring loudness, with a level meter analyzer + * @type {MediaStreamSourceNode} + */ + this.mic = null; + } + + /** + * Get the current loudness of sound received by the microphone. + * Sound is measured in RMS and smoothed. + * Some code adapted from Tone.js: https://github.com/Tonejs/Tone.js + * @return {number} loudness scaled 0 to 100 + */ + getLoudness () { + // The microphone has not been set up, so try to connect to it + if (!this.mic && !this.connectingToMic) { + this.connectingToMic = true; // prevent multiple connection attempts + navigator.mediaDevices.getUserMedia({audio: true}).then(stream => { + this.audioStream = stream; + this.mic = this.audioContext.createMediaStreamSource(stream); + this.analyser = this.audioContext.createAnalyser(); + this.mic.connect(this.analyser); + this.micDataArray = new Float32Array(this.analyser.fftSize); + }) + .catch(err => { + log.warn(err); + }); + } + + // If the microphone is set up and active, measure the loudness + if (this.mic && this.audioStream.active) { + this.analyser.getFloatTimeDomainData(this.micDataArray); + let sum = 0; + // compute the RMS of the sound + for (let i = 0; i < this.micDataArray.length; i++){ + sum += Math.pow(this.micDataArray[i], 2); + } + let rms = Math.sqrt(sum / this.micDataArray.length); + // smooth the value, if it is descending + if (this._lastValue) { + rms = Math.max(rms, this._lastValue * 0.6); + } + this._lastValue = rms; + + // Scale the measurement so it's more sensitive to quieter sounds + rms *= 1.63; + rms = Math.sqrt(rms); + // Scale it up to 0-100 and round + rms = Math.round(rms * 100); + // Prevent it from going above 100 + rms = Math.min(rms, 100); + return rms; + } + + // if there is no microphone input, return -1 + return -1; + } +} + +module.exports = Loudness; diff --git a/src/index.js b/src/index.js index 95f3bbe..cf52312 100644 --- a/src/index.js +++ b/src/index.js @@ -1,337 +1,9 @@ -const StartAudioContext = require('startaudiocontext'); -const AudioContext = require('audio-context'); - -const log = require('./log'); -const uid = require('./uid'); - -const PitchEffect = require('./effects/PitchEffect'); -const PanEffect = require('./effects/PanEffect'); - -const SoundPlayer = require('./SoundPlayer'); -const ADPCMSoundDecoder = require('./ADPCMSoundDecoder'); - /** - * @fileOverview Scratch Audio is divided into a single AudioEngine, - * that handles global functionality, and AudioPlayers, belonging to individual sprites and clones. + * @fileOverview Scratch Audio is divided into a single AudioEngine, that + * handles global functionality, and AudioPlayers, belonging to individual + * sprites and clones. */ -/** - * Wrapper to ensure that audioContext.decodeAudioData is a promise - * @param {object} audioContext The current AudioContext - * @param {ArrayBuffer} buffer Audio data buffer to decode - * @return {Promise} A promise that resolves to the decoded audio - */ -const decodeAudioData = function (audioContext, buffer) { - // Check for newer promise-based API - if (audioContext.decodeAudioData.length === 1) { - return audioContext.decodeAudioData(buffer); - } - // Fall back to callback API - return new Promise((resolve, reject) => { - audioContext.decodeAudioData(buffer, - decodedAudio => resolve(decodedAudio), - error => reject(error) - ); - }); -}; - - -class AudioPlayer { - /** - * Each sprite or clone has an audio player - * the audio player handles sound playback, volume, and the sprite-specific audio effects: - * pitch and pan - * @param {AudioEngine} audioEngine AudioEngine for player - * @constructor - */ - constructor (audioEngine) { - this.audioEngine = audioEngine; - - // Create the audio effects - this.pitchEffect = new PitchEffect(); - this.panEffect = new PanEffect(this.audioEngine); - - // Chain the audio effects together - // effectsNode -> panEffect -> audioEngine.input - this.effectsNode = this.audioEngine.audioContext.createGain(); - this.effectsNode.connect(this.panEffect.input); - this.panEffect.connect(this.audioEngine.input); - - // reset effects to their default parameters - this.clearEffects(); - - // sound players that are currently playing, indexed by the sound's soundId - this.activeSoundPlayers = {}; - } - - /** - * Get this sprite's input node, so that other objects can route sound through it. - * @return {AudioNode} the AudioNode for this sprite's input - */ - getInputNode () { - return this.effectsNode; - } - - /** - * Play a sound - * @param {string} soundId - the soundId id of a sound file - * @return {Promise} a Promise that resolves when the sound finishes playing - */ - playSound (soundId) { - // if this sound is not in the audio engine, return - if (!this.audioEngine.audioBuffers[soundId]) { - return; - } - - // if this sprite or clone is already playing this sound, stop it first - if (this.activeSoundPlayers[soundId]) { - this.activeSoundPlayers[soundId].stop(); - } - - // create a new soundplayer to play the sound - const player = new SoundPlayer(this.audioEngine.audioContext); - player.setBuffer(this.audioEngine.audioBuffers[soundId]); - player.connect(this.effectsNode); - this.pitchEffect.updatePlayer(player); - player.start(); - - // add it to the list of active sound players - this.activeSoundPlayers[soundId] = player; - - // remove sounds that are not playing from the active sound players array - for (const id in this.activeSoundPlayers) { - if (this.activeSoundPlayers.hasOwnProperty(id)) { - if (!this.activeSoundPlayers[id].isPlaying) { - delete this.activeSoundPlayers[id]; - } - } - } - - return player.finished(); - } - - /** - * Stop all sounds that are playing - */ - stopAllSounds () { - // stop all active sound players - for (const soundId in this.activeSoundPlayers) { - this.activeSoundPlayers[soundId].stop(); - } - } - - /** - * Set an audio effect to a value - * @param {string} effect - the name of the effect - * @param {number} value - the value to set the effect to - */ - setEffect (effect, value) { - switch (effect) { - case this.audioEngine.EFFECT_NAMES.pitch: - this.pitchEffect.set(value, this.activeSoundPlayers); - break; - case this.audioEngine.EFFECT_NAMES.pan: - this.panEffect.set(value); - break; - } - } - - /** - * Clear all audio effects - */ - clearEffects () { - this.panEffect.set(0); - this.pitchEffect.set(0, this.activeSoundPlayers); - if (this.audioEngine === null) return; - this.effectsNode.gain.setTargetAtTime(1.0, 0, this.audioEngine.DECAY_TIME); - } - - /** - * Set the volume for sounds played by this AudioPlayer - * @param {number} value - the volume in range 0-100 - */ - setVolume (value) { - if (this.audioEngine === null) return; - this.effectsNode.gain.setTargetAtTime(value / 100, 0, this.audioEngine.DECAY_TIME); - } - - /** - * Clean up and disconnect audio nodes. - */ - dispose () { - this.panEffect.dispose(); - this.effectsNode.disconnect(); - } -} - - -/** - * There is a single instance of the AudioEngine. It handles global audio properties and effects, - * loads all the audio buffers for sounds belonging to sprites. - */ -class AudioEngine { - constructor () { - this.audioContext = new AudioContext(); - StartAudioContext(this.audioContext); - - this.input = this.audioContext.createGain(); - this.input.connect(this.audioContext.destination); - - // a map of soundIds to audio buffers, holding sounds for all sprites - this.audioBuffers = {}; - - // microphone, for measuring loudness, with a level meter analyzer - this.mic = null; - } - - /** - * Names of the audio effects. - * @enum {string} - */ - get EFFECT_NAMES () { - return { - pitch: 'pitch', - pan: 'pan' - }; - } - - /** - * A short duration, for use as a time constant for exponential audio parameter transitions. - * See: - * https://developer.mozilla.org/en-US/docs/Web/API/AudioParam/setTargetAtTime - * @const {number} - */ - get DECAY_TIME () { - return 0.001; - } - - /** - * Decode a sound, decompressing it into audio samples. - * Store a reference to it the sound in the audioBuffers dictionary, indexed by soundId - * @param {object} sound - an object containing audio data and metadata for a sound - * @property {Buffer} data - sound data loaded from scratch-storage. - * @returns {?Promise} - a promise which will resolve to the soundId if decoded and stored. - */ - decodeSound (sound) { - // Make a copy of the buffer because decoding detaches the original buffer - const bufferCopy1 = sound.data.buffer.slice(0); - - const soundId = uid(); - // Partially apply updateSoundBuffer function with the current - // soundId so that it's ready to be used on successfully decoded audio - const addDecodedAudio = this.updateSoundBuffer.bind(this, soundId); - - // Attempt to decode the sound using the browser's native audio data decoder - // If that fails, attempt to decode as ADPCM - return decodeAudioData(this.audioContext, bufferCopy1).then( - addDecodedAudio, - () => { - // The audio context failed to parse the sound data - // we gave it, so try to decode as 'adpcm' - - // First we need to create another copy of our original data - const bufferCopy2 = sound.data.buffer.slice(0); - // Try decoding as adpcm - return (new ADPCMSoundDecoder(this.audioContext)).decode(bufferCopy2) - .then( - addDecodedAudio, - error => { - log.warn('audio data could not be decoded', error); - } - ); - } - ); - } - - /** - * Retrieve the audio buffer as held in memory for a given sound id. - * @param {!string} soundId - the id of the sound buffer to get - * @return {AudioBuffer} the buffer corresponding to the given sound id. - */ - getSoundBuffer (soundId) { - return this.audioBuffers[soundId]; - } - - /** - * Add or update the in-memory audio buffer to a new one by soundId. - * @param {!string} soundId - the id of the sound buffer to update. - * @param {AudioBuffer} newBuffer - the new buffer to swap in. - * @return {string} The uid of the sound that was updated or added - */ - updateSoundBuffer (soundId, newBuffer) { - this.audioBuffers[soundId] = newBuffer; - return soundId; - } - - /** - * An older version of the AudioEngine had this function to load all sounds - * This is a stub to provide a warning when it is called - * @todo remove this - */ - loadSounds () { - log.warn('The loadSounds function is no longer available. Please use Scratch Storage.'); - } - - /** - * Get the current loudness of sound received by the microphone. - * Sound is measured in RMS and smoothed. - * Some code adapted from Tone.js: https://github.com/Tonejs/Tone.js - * @return {number} loudness scaled 0 to 100 - */ - getLoudness () { - // The microphone has not been set up, so try to connect to it - if (!this.mic && !this.connectingToMic) { - this.connectingToMic = true; // prevent multiple connection attempts - navigator.mediaDevices.getUserMedia({audio: true}).then(stream => { - this.audioStream = stream; - this.mic = this.audioContext.createMediaStreamSource(stream); - this.analyser = this.audioContext.createAnalyser(); - this.mic.connect(this.analyser); - this.micDataArray = new Float32Array(this.analyser.fftSize); - }) - .catch(err => { - log.warn(err); - }); - } - - // If the microphone is set up and active, measure the loudness - if (this.mic && this.audioStream.active) { - this.analyser.getFloatTimeDomainData(this.micDataArray); - let sum = 0; - // compute the RMS of the sound - for (let i = 0; i < this.micDataArray.length; i++){ - sum += Math.pow(this.micDataArray[i], 2); - } - let rms = Math.sqrt(sum / this.micDataArray.length); - // smooth the value, if it is descending - if (this._lastValue) { - rms = Math.max(rms, this._lastValue * 0.6); - } - this._lastValue = rms; - - // Scale the measurement so it's more sensitive to quieter sounds - rms *= 1.63; - rms = Math.sqrt(rms); - // Scale it up to 0-100 and round - rms = Math.round(rms * 100); - // Prevent it from going above 100 - rms = Math.min(rms, 100); - return rms; - } - - // if there is no microphone input, return -1 - return -1; - } - - /** - * Create an AudioPlayer. Each sprite or clone has an AudioPlayer. - * It includes a reference to the AudioEngine so it can use global - * functionality such as playing notes. - * @return {AudioPlayer} new AudioPlayer instance - */ - createPlayer () { - return new AudioPlayer(this); - } -} +const AudioEngine = require('./AudioEngine'); module.exports = AudioEngine;