scratchfoundation/scratch-vm
1
0
Fork 0
mirror of https://github.com/scratchfoundation/scratch-vm.git synced 2024-12-24 15:02:52 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

use scratch-audio EffectChain and SoundPlayer in music extension

Browse source 
- Use AudioEngine to decode sounds
- Store players instead of buffers
- Use SoundPlayer stop event to track concurrency
This commit is contained in:
Michael "Z" Goddard 2018-06-20 14:33:39 -04:00
parent 35366d90d1
commit 7e48bed0ab
No known key found for this signature in database
GPG key ID: 762CD40DD5349872
1 changed files with 87 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

151
src/extensions/scratch3_music/index.js
View file

@ -52,18 +52,25 @@ class Scratch3MusicBlocks {
this._concurrencyCounter = 0; this._concurrencyCounter = 0;
/** /**
* An array of audio buffers, one for each drum sound. * An array of sound players, one for each drum sound.
* @type {Array} * @type {Array}
* @private * @private
*/ */
this._drumBuffers = []; this._drumPlayers = [];
/** /**
* An array of arrays of audio buffers. Each instrument has one or more audio buffers. * An array of arrays of sound players. Each instrument has one or more audio players.
* @type {Array[]} * @type {Array[]}
* @private * @private
*/ */
this._instrumentBufferArrays = []; this._instrumentPlayerArrays = [];
/**
* An array of arrays of sound players. Each instrument mya have an audio player for each playable note.
* @type {Array[]}
* @private
*/
this._instrumentPlayerNoteArrays = [];
/** /**
* An array of audio bufferSourceNodes. Each time you play an instrument or drum sound, * An array of audio bufferSourceNodes. Each time you play an instrument or drum sound,
@ -87,14 +94,15 @@ class Scratch3MusicBlocks {
const loadingPromises = []; const loadingPromises = [];
this.DRUM_INFO.forEach((drumInfo, index) => { this.DRUM_INFO.forEach((drumInfo, index) => {
const filePath = `drums/${drumInfo.fileName}`; const filePath = `drums/${drumInfo.fileName}`;
const promise = this._storeSound(filePath, index, this._drumBuffers); const promise = this._storeSound(filePath, index, this._drumPlayers);
loadingPromises.push(promise); loadingPromises.push(promise);
}); });
this.INSTRUMENT_INFO.forEach((instrumentInfo, instrumentIndex) => { this.INSTRUMENT_INFO.forEach((instrumentInfo, instrumentIndex) => {
this._instrumentBufferArrays[instrumentIndex] = []; this._instrumentPlayerArrays[instrumentIndex] = [];
this._instrumentPlayerNoteArrays[instrumentIndex] = [];
instrumentInfo.samples.forEach((sample, noteIndex) => { instrumentInfo.samples.forEach((sample, noteIndex) => {
const filePath = `instruments/${instrumentInfo.dirName}/${sample}`; const filePath = `instruments/${instrumentInfo.dirName}/${sample}`;
const promise = this._storeSound(filePath, noteIndex, this._instrumentBufferArrays[instrumentIndex]); const promise = this._storeSound(filePath, noteIndex, this._instrumentPlayerArrays[instrumentIndex]);
loadingPromises.push(promise); loadingPromises.push(promise);
}); });
}); });
@ -104,22 +112,22 @@ class Scratch3MusicBlocks {
} }
/** /**
* Decode a sound and store the buffer in an array. * Decode a sound and store the player in an array.
* @param {string} filePath - the audio file name. * @param {string} filePath - the audio file name.
* @param {number} index - the index at which to store the audio buffer. * @param {number} index - the index at which to store the audio player.
* @param {array} bufferArray - the array of buffers in which to store it. * @param {array} playerArray - the array of players in which to store it.
* @return {Promise} - a promise which will resolve once the sound has been stored. * @return {Promise} - a promise which will resolve once the sound has been stored.
*/ */
_storeSound (filePath, index, bufferArray) { _storeSound (filePath, index, playerArray) {
const fullPath = `${filePath}.mp3`; const fullPath = `${filePath}.mp3`;
if (!assetData[fullPath]) return; if (!assetData[fullPath]) return;
// The sound buffer has already been downloaded via the manifest file required above. // The sound player has already been downloaded via the manifest file required above.
const soundBuffer = assetData[fullPath]; const soundBuffer = assetData[fullPath];
return this._decodeSound(soundBuffer).then(buffer => { return this._decodeSound(soundBuffer).then(player => {
bufferArray[index] = buffer; playerArray[index] = player;
}); });
} }
@ -129,24 +137,14 @@ class Scratch3MusicBlocks {
* @return {Promise} - a promise which will resolve once the sound has decoded. * @return {Promise} - a promise which will resolve once the sound has decoded.
*/ */
_decodeSound (soundBuffer) { _decodeSound (soundBuffer) {
const context = this.runtime.audioEngine && this.runtime.audioEngine.audioContext; const engine = this.runtime.audioEngine;
if (!context) { if (!engine) {
return Promise.reject(new Error('No Audio Context Detected')); return Promise.reject(new Error('No Audio Context Detected'));
} }
// Check for newer promise-based API // Check for newer promise-based API
if (context.decodeAudioData.length === 1) { return engine.decodeSoundPlayer({data: {buffer: soundBuffer}});
return context.decodeAudioData(soundBuffer);
} else { // eslint-disable-line no-else-return
// Fall back to callback API
return new Promise((resolve, reject) =>
context.decodeAudioData(soundBuffer,
buffer => resolve(buffer),
error => reject(error)
)
);
}
} }
/** /**
@ -774,26 +772,34 @@ class Scratch3MusicBlocks {
*/ */
_playDrumNum (util, drumNum) { _playDrumNum (util, drumNum) {
if (util.runtime.audioEngine === null) return; if (util.runtime.audioEngine === null) return;
if (util.target.audioPlayer === null) return; if (util.target.sprite.soundBank === null) return;
// If we're playing too many sounds, do not play the drum sound. // If we're playing too many sounds, do not play the drum sound.
if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) { if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) {
return; return;
} }
const outputNode = util.target.audioPlayer.getInputNode();
const context = util.runtime.audioEngine.audioContext;
const bufferSource = context.createBufferSource();
bufferSource.buffer = this._drumBuffers[drumNum];
bufferSource.connect(outputNode);
bufferSource.start();
const bufferSourceIndex = this._bufferSources.length; const player = this._drumPlayers[drumNum];
this._bufferSources.push(bufferSource);
if (typeof player === 'undefined') return;
if (player.isPlaying) {
// Take the internal player state and create a new player with it.
// `.play` does this internally but then instructs the sound to
// stop.
player.take();
}
const engine = util.runtime.audioEngine;
const chain = engine.createEffectChain();
chain.setEffectsFromTarget(util.target);
player.connect(chain);
this._concurrencyCounter++; this._concurrencyCounter++;
bufferSource.onended = () => { player.once('stop', () => {
this._concurrencyCounter--; this._concurrencyCounter--;
delete this._bufferSources[bufferSourceIndex]; });
};
player.play();
} }
/** /**
@ -852,7 +858,7 @@ class Scratch3MusicBlocks {
*/ */
_playNote (util, note, durationSec) { _playNote (util, note, durationSec) {
if (util.runtime.audioEngine === null) return; if (util.runtime.audioEngine === null) return;
if (util.target.audioPlayer === null) return; if (util.target.sprite.soundBank === null) return;
// If we're playing too many sounds, do not play the note. // If we're playing too many sounds, do not play the note.
if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) { if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) {
@ -867,28 +873,37 @@ class Scratch3MusicBlocks {
const sampleIndex = this._selectSampleIndexForNote(note, sampleArray); const sampleIndex = this._selectSampleIndexForNote(note, sampleArray);
// If the audio sample has not loaded yet, bail out // If the audio sample has not loaded yet, bail out
if (typeof this._instrumentBufferArrays[inst] === 'undefined') return; if (typeof this._instrumentPlayerArrays[inst] === 'undefined') return;
if (typeof this._instrumentBufferArrays[inst][sampleIndex] === 'undefined') return; if (typeof this._instrumentPlayerArrays[inst][sampleIndex] === 'undefined') return;
// Create the audio buffer to play the note, and set its pitch // Fetch the sound player to play the note.
const context = util.runtime.audioEngine.audioContext; const engine = util.runtime.audioEngine;
const bufferSource = context.createBufferSource();
const bufferSourceIndex = this._bufferSources.length; if (!this._instrumentPlayerNoteArrays[inst][note]) {
this._bufferSources.push(bufferSource); this._instrumentPlayerNoteArrays[inst][note] = this._instrumentPlayerArrays[inst][sampleIndex].take();
}
bufferSource.buffer = this._instrumentBufferArrays[inst][sampleIndex]; const player = this._instrumentPlayerNoteArrays[inst][note];
if (player.isPlaying) {
// Take the internal player state and create a new player with it.
// `.play` does this internally but then instructs the sound to
// stop.
player.take();
}
const chain = engine.createEffectChain();
chain.setEffectsFromTarget(util.target);
// Set its pitch.
const sampleNote = sampleArray[sampleIndex]; const sampleNote = sampleArray[sampleIndex];
bufferSource.playbackRate.value = this._ratioForPitchInterval(note - sampleNote); const notePitchInterval = this._ratioForPitchInterval(note - sampleNote);
// Create a gain node for this note, and connect it to the sprite's audioPlayer. // Create a gain node for this note, and connect it to the sprite's
const gainNode = context.createGain(); // simulated effectChain.
bufferSource.connect(gainNode); const context = engine.audioContext;
const outputNode = util.target.audioPlayer.getInputNode(); const releaseGain = context.createGain();
gainNode.connect(outputNode); releaseGain.connect(chain.getInputNode());
// Start playing the note
bufferSource.start();
// Schedule the release of the note, ramping its gain down to zero, // Schedule the release of the note, ramping its gain down to zero,
// and then stopping the sound. // and then stopping the sound.
@ -898,16 +913,24 @@ class Scratch3MusicBlocks {
} }
const releaseStart = context.currentTime + durationSec; const releaseStart = context.currentTime + durationSec;
const releaseEnd = releaseStart + releaseDuration; const releaseEnd = releaseStart + releaseDuration;
gainNode.gain.setValueAtTime(1, releaseStart); releaseGain.gain.setValueAtTime(1, releaseStart);
gainNode.gain.linearRampToValueAtTime(0.0001, releaseEnd); releaseGain.gain.linearRampToValueAtTime(0.0001, releaseEnd);
bufferSource.stop(releaseEnd);
// Update the concurrency counter
this._concurrencyCounter++; this._concurrencyCounter++;
bufferSource.onended = () => { player.once('stop', () => {
this._concurrencyCounter--; this._concurrencyCounter--;
delete this._bufferSources[bufferSourceIndex]; });
};
// Start playing the note
player.play();
// Connect the player to the gain node.
player.connect({getInputNode () {
return releaseGain;
}});
// Set playback now after play creates the outputNode.
player.outputNode.playbackRate.value = notePitchInterval;
// Schedule playback to stop.
player.outputNode.stop(releaseEnd);
} }
/** /**