Comment all the things!

This commit is contained in:
Eric Rosenbaum 2017-02-02 14:53:17 -05:00
parent cdc49def15
commit 1a26fecca1
13 changed files with 239 additions and 77 deletions

View file

@ -1,19 +1,24 @@
/*
ADPCMSoundLoader loads wav files that have been compressed with the ADPCM format
based on code from Scratch-Flash:
https://github.com/LLK/scratch-flash/blob/master/src/sound/WAVFile.as
*/
var ArrayBufferStream = require('./ArrayBufferStream');
var Tone = require('tone');
var log = require('./log');
/*
* Load wav audio files that have been compressed with the ADPCM format.
* This is necessary because, while web browsers have native decoders for many audio
* formats, ADPCM is a non-standard format used by Scratch since its early days.
* This decoder is based on code from Scratch-Flash:
* https://github.com/LLK/scratch-flash/blob/master/src/sound/WAVFile.as
* @constructor
*/
function ADPCMSoundLoader () {
}
/**
* Load an ADPCM sound file from a URL, decode it, and return a promise
* with the audio buffer.
* @param {string} url - a url pointing to the ADPCM wav file
* @return {Tone.Buffer}
*/
ADPCMSoundLoader.prototype.load = function (url) {
return new Promise(function (resolve, reject) {
@ -73,7 +78,10 @@ ADPCMSoundLoader.prototype.load = function (url) {
}.bind(this));
};
/**
* Data used by the decompression algorithm
* @type {Array}
*/
ADPCMSoundLoader.prototype.stepTable = [
7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
50, 55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230,
@ -82,10 +90,20 @@ ADPCMSoundLoader.prototype.stepTable = [
3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487,
12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767];
/**
* Data used by the decompression algorithm
* @type {Array}
*/
ADPCMSoundLoader.prototype.indexTable = [
-1, -1, -1, -1, 2, 4, 6, 8,
-1, -1, -1, -1, 2, 4, 6, 8];
/**
* Extract a chunk of audio data from the stream, consisting of a set of audio data bytes
* @param {string} chunkType - the type of chunk to extract. 'data' or 'fmt' (format)
* @param {ArrayBufferStream} stream - an stream containing the audio data
* @return {ArrayBufferStream} a stream containing the desired chunk
*/
ADPCMSoundLoader.prototype.extractChunk = function (chunkType, stream) {
stream.position = 12;
while (stream.position < (stream.getLength() - 8)) {
@ -100,9 +118,14 @@ ADPCMSoundLoader.prototype.extractChunk = function (chunkType, stream) {
}
};
/**
* Decompress sample data using the IMA ADPCM algorithm.
* Note: Handles only one channel, 4-bits per sample.
* @param {ArrayBufferStream} compressedData - a stream of compressed audio samples
* @param {number} blockSize - the number of bytes in the stream
* @return {Int16Array} the uncompressed audio samples
*/
ADPCMSoundLoader.prototype.imaDecompress = function (compressedData, blockSize) {
// Decompress sample data using the IMA ADPCM algorithm.
// Note: Handles only one channel, 4-bits/sample.
var sample, step, code, delta;
var index = 0;
var lastByte = -1; // -1 indicates that there is no saved lastByte

View file

@ -1,39 +1,59 @@
/*
ArrayBufferStream wraps the built-in javascript ArrayBuffer, adding the ability to access
data in it like a stream. You can request to read a value from the front of the array,
such as an 8 bit unsigned int, a 16 bit int, etc, and it will keep track of the position
within the byte array, so that successive reads are consecutive.
*/
/**
* ArrayBufferStream wraps the built-in javascript ArrayBuffer, adding the ability to access
* data in it like a stream, tracking its position.
* You can request to read a value from the front of the array, and it will keep track of the position
* within the byte array, so that successive reads are consecutive.
* The available types to read include:
* Uint8, Uint8String, Int16, Uint16, Int32, Uint32
* @param {ArrayBuffer} arrayBuffer - array to use as a stream
* @constructor
*/
function ArrayBufferStream (arrayBuffer) {
this.arrayBuffer = arrayBuffer;
this.position = 0;
}
// return a new ArrayBufferStream that is a slice of the existing one
/**
* Return a new ArrayBufferStream that is a slice of the existing one
* @param {number} length - the number of bytes of extract
* @return {ArrayBufferStream} the extracted stream
*/
ArrayBufferStream.prototype.extract = function (length) {
var slicedArrayBuffer = this.arrayBuffer.slice(this.position, this.position+length);
var newStream = new ArrayBufferStream(slicedArrayBuffer);
return newStream;
};
/**
* @return {number} the length of the stream in bytes
*/
ArrayBufferStream.prototype.getLength = function () {
return this.arrayBuffer.byteLength;
};
/**
* @return {number} the number of bytes available after the current position in the stream
*/
ArrayBufferStream.prototype.getBytesAvailable = function () {
return (this.arrayBuffer.byteLength - this.position);
};
/**
* Read an unsigned 8 bit integer from the stream
* @return {number}
*/
ArrayBufferStream.prototype.readUint8 = function () {
var val = new Uint8Array(this.arrayBuffer, this.position, 1)[0];
this.position += 1;
return val;
};
// convert a sequence of bytes of the given length to a string
// for small length strings only
/**
* Read a sequence of bytes of the given length and convert to a string.
* This is a convenience method for use with short strings.
* @param {number} length - the number of bytes to convert
* @return {String} a String made by concatenating the chars in the input
*/
ArrayBufferStream.prototype.readUint8String = function (length) {
var arr = new Uint8Array(this.arrayBuffer, this.position, length);
this.position += length;
@ -44,24 +64,40 @@ ArrayBufferStream.prototype.readUint8String = function (length) {
return str;
};
/**
* Read a 16 bit integer from the stream
* @return {number}
*/
ArrayBufferStream.prototype.readInt16 = function () {
var val = new Int16Array(this.arrayBuffer, this.position, 1)[0];
this.position += 2; // one 16 bit int is 2 bytes
return val;
};
/**
* Read an unsigned 16 bit integer from the stream
* @return {number}
*/
ArrayBufferStream.prototype.readUint16 = function () {
var val = new Uint16Array(this.arrayBuffer, this.position, 1)[0];
this.position += 2; // one 16 bit int is 2 bytes
return val;
};
/**
* Read a 32 bit integer from the stream
* @return {number}
*/
ArrayBufferStream.prototype.readInt32 = function () {
var val = new Int32Array(this.arrayBuffer, this.position, 1)[0];
this.position += 4; // one 32 bit int is 4 bytes
return val;
};
/**
* Read an unsigned 32 bit integer from the stream
* @return {number}
*/
ArrayBufferStream.prototype.readUint32 = function () {
var val = new Uint32Array(this.arrayBuffer, this.position, 1)[0];
this.position += 4; // one 32 bit int is 4 bytes

View file

@ -1,6 +1,11 @@
var SoundPlayer = require('./SoundPlayer');
var Tone = require('tone');
/**
* A prototype for the drum sound functionality that can load drum sounds, play, and stop them.
* @param {Tone.Gain} outputNode - a webAudio node that the drum sounds will send their output to
* @constructor
*/
function DrumPlayer (outputNode) {
this.outputNode = outputNode;
@ -35,11 +40,21 @@ function DrumPlayer (outputNode) {
}
}
/**
* Play a drum sound.
* The parameter for output node allows sprites or clones to send the drum sound
* to their individual audio effect chains.
* @param {number} drum - the drum number to play (0-indexed)
* @param {Tone.Gain} outputNode - a node to send the output to
*/
DrumPlayer.prototype.play = function (drum, outputNode) {
this.drumSounds[drum].outputNode = outputNode;
this.drumSounds[drum].start();
};
/**
* Stop all drum sounds.
*/
DrumPlayer.prototype.stopAll = function () {
for (var i=0; i<this.drumSounds.length; i++) {
this.drumSounds[i].stop();

View file

@ -1,10 +1,21 @@
var Tone = require('tone');
var Soundfont = require('soundfont-player');
/**
* A prototype for the instrument sound functionality that can play notes.
* This prototype version (which will be replaced at some point) uses an
* existing soundfont library that creates several limitations:
* The sound files are high quality but large, so they are loaded 'on demand,' at the time the
* play note or set instrument block runs, causing a delay of a few seconds.
* Using this library we don't have a way to set the volume, sustain the note beyond the sample
* duration, or run it through the sprite-specific audio effects.
* @param {Tone.Gain} outputNode - a webAudio node that the instrument will send its output to
* @constructor
*/
function InstrumentPlayer (outputNode) {
this.outputNode = outputNode;
// instrument names used by Musyng Kite soundfont, in order to
// Instrument names used by Musyng Kite soundfont, in order to
// match scratch instruments
this.instrumentNames = ['acoustic_grand_piano', 'electric_piano_1',
'drawbar_organ', 'acoustic_guitar_nylon', 'electric_guitar_clean',
@ -15,6 +26,15 @@ function InstrumentPlayer (outputNode) {
this.instruments = [];
}
/**
* Play a note for some number of seconds with a particular instrument.
* Load the instrument first, if it has not already been loaded.
* The duration is in seconds because the AudioEngine manages the tempo,
* and converts beats to seconds.
* @param {number} note - a MIDI note number
* @param {number} sec - a duration in seconds
* @param {number} instrumentNum - an instrument number (0-indexed)
*/
InstrumentPlayer.prototype.playNoteForSecWithInst = function (note, sec, instrumentNum) {
this.loadInstrument(instrumentNum)
.then(() => {
@ -24,6 +44,11 @@ InstrumentPlayer.prototype.playNoteForSecWithInst = function (note, sec, instrum
});
};
/**
* Load an instrument by number
* @param {number} instrumentNum - an instrument number (0-indexed)
* @return {Promise} a Promise that resolves once the instrument audio data has been loaded
*/
InstrumentPlayer.prototype.loadInstrument = function (instrumentNum) {
if (this.instruments[instrumentNum]) {
return Promise.resolve();
@ -36,6 +61,9 @@ InstrumentPlayer.prototype.loadInstrument = function (instrumentNum) {
}
};
/**
* Stop all notes being played on all instruments
*/
InstrumentPlayer.prototype.stopAll = function () {
for (var i=0; i<this.instruments.length; i++) {
if (this.instruments[i]) {

View file

@ -1,6 +1,10 @@
var Tone = require('tone');
var log = require('./log');
/**
* A SoundPlayer stores an audio buffer, and plays it
* @constructor
*/
function SoundPlayer () {
this.outputNode = null;
this.buffer = new Tone.Buffer();
@ -8,14 +12,27 @@ function SoundPlayer () {
this.playbackRate = 1;
this.isPlaying = false;
}
/**
* Connect the SoundPlayer to an output node
* @param {Tone.Gain} node - an output node to connect to
*/
SoundPlayer.prototype.connect = function (node) {
this.outputNode = node;
};
/**
* Set an audio buffer
* @param {Tone.Buffer} buffer
*/
SoundPlayer.prototype.setBuffer = function (buffer) {
this.buffer = buffer;
};
/**
* Set the playback rate for the sound
* @param {number} playbackRate - a ratio where 1 is normal playback, 0.5 is half speed, 2 is double speed, etc.
*/
SoundPlayer.prototype.setPlaybackRate = function (playbackRate) {
this.playbackRate = playbackRate;
if (this.bufferSource && this.bufferSource.playbackRate) {
@ -23,6 +40,9 @@ SoundPlayer.prototype.setPlaybackRate = function (playbackRate) {
}
};
/**
* Stop the sound
*/
SoundPlayer.prototype.stop = function () {
if (this.bufferSource) {
this.bufferSource.stop();
@ -30,6 +50,10 @@ SoundPlayer.prototype.stop = function () {
this.isPlaying = false;
};
/**
* Start playing the sound
* The web audio framework requires a new audio buffer source node for each playback
*/
SoundPlayer.prototype.start = function () {
if (!this.buffer || !this.buffer.loaded) {
log.warn('tried to play a sound that was not loaded yet');
@ -44,6 +68,11 @@ SoundPlayer.prototype.start = function () {
this.isPlaying = true;
};
/**
* The sound has finished playing. This is called at the correct time even if the playback rate
* has been changed
* @return {Promise} a Promise that resolves when the sound finishes playing
*/
SoundPlayer.prototype.finished = function () {
var storedContext = this;
return new Promise(function (resolve) {

View file

@ -1,16 +1,11 @@
var Tone = require('tone');
/**
* @fileoverview
* An echo effect (aka 'delay effect' in audio terms)
* Effect value of 0 mutes the effect
* Values up to 100 set the echo feedback amount,
* increasing the time it takes the echo to fade away
* Clamped 0-100
*/
/**
* Initialize and chain the effect
* @constructor
*/
function EchoEffect () {

View file

@ -1,15 +1,10 @@
var Tone = require('tone');
/**
* @fileoverview
* An fuzz effect (aka 'distortion effect' in audio terms)
* A fuzz effect (aka 'distortion effect' in audio terms)
* Effect value controls the wet/dry amount:
* 0 passes through none of the effect, 100 passes through all effect
* Clamped 0-100
*/
/**
* Initialize and chain the effect
* @constructor
*/
function FuzzEffect () {

View file

@ -1,15 +1,10 @@
var Tone = require('tone');
/**
* @fileoverview
* An pan effect, which moves the sound to the left or right between the speakers
* A pan effect, which moves the sound to the left or right between the speakers
* Effect value of -100 puts the audio entirely on the left channel,
* 0 centers it, 100 puts it on the right.
* Clamped -100 to 100
*/
/**
* Initialize and chain the effect
* @constructor
*/
function PanEffect () {

View file

@ -1,8 +1,7 @@
var Tone = require('tone');
/**
* @fileoverview
* An pitch change effect, which changes the playback rate of the sound in order
* A pitch change effect, which changes the playback rate of the sound in order
* to change its pitch: reducing the playback rate lowers the pitch, increasing the rate
* raises the pitch. The duration of the sound is also changed.
*
@ -17,10 +16,6 @@ var Tone = require('tone');
* Note that this effect functions differently from the other audio effects. It is
* not part of a chain of audio nodes. Instead, it provides a way to set the playback
* on one SoundPlayer or a group of them.
*/
/**
* Initialize the effect
* @constructor
*/
function PitchEffect () {

View file

@ -1,15 +1,10 @@
var Tone = require('tone');
/**
* @fileoverview
* A reverb effect, simulating reverberation in a room
* Effect value controls the wet/dry amount:
* 0 passes through none of the effect, 100 passes through all effect
* Clamped 0 to 100
*/
/**
* Initialize and chain the effect
* @constructor
*/
function ReverbEffect () {

View file

@ -2,7 +2,6 @@
var Tone = require('tone');
/**
* @fileoverview
* A "robotic" effect that adds a low-pitched buzzing to the sound, reminiscent of the
* voice of the daleks from Dr. Who.
* In audio terms it is a feedback comb filter with a short delay time.
@ -11,10 +10,6 @@ var Tone = require('tone');
* Other values change the pitch of the effect, in units of 10 steps per semitone.
* The effect value is not clamped (but probably should be).
* Exterminate.
*/
/**
* Initialize and chain the effect
* @constructor
*/
function RoboticEffect () {

View file

@ -1,7 +1,6 @@
var Tone = require('tone');
/**
* @fileoverview
* A wobble effect. In audio terms, it sounds like tremolo.
* It is implemented using a low frequency oscillator (LFO) controlling
* a gain node, which causes the loudness of the signal passing through
@ -10,10 +9,6 @@ var Tone = require('tone');
* 0 passes through none of the effect, 100 passes through all effect
* Effect value also controls the frequency of the LFO.
* Clamped 0 to 100
*/
/**
* Initialize and chain the effect
* @constructor
*/
function WobbleEffect () {

View file

@ -14,14 +14,17 @@ var ADPCMSoundLoader = require('./ADPCMSoundLoader');
var InstrumentPlayer = require('./InstrumentPlayer');
var DrumPlayer = require('./DrumPlayer');
/* Audio Engine
The Scratch runtime has a single audio engine that handles global audio properties and effects,
loads all the audio buffers for sounds belonging to sprites, and creates a single instrument player
and a drum player, used by all play note and play drum blocks
*/
/**
* @fileOverview Scratch Audio is divided into a single AudioEngine,
* that handles global functionality, and AudioPlayers belonging to individual sprites and clones.
*/
/**
* There is a single instance of the AudioEngine. It handles global audio properties and effects,
* loads all the audio buffers for sounds belonging to sprites, and creates a single instrument player
* and a drum player, used by all play note and play drum blocks.
* @constructor
*/
function AudioEngine () {
// create the global audio effects
@ -52,8 +55,11 @@ function AudioEngine () {
this.audioBuffers = {};
}
/**
* Load all sounds for a sprite and store them in the audioBuffers dictionary, indexed by md5
* @param {Object} sounds - an array of objects containing metadata for sound files of a sprite
*/
AudioEngine.prototype.loadSounds = function (sounds) {
// most sounds decode natively, but for adpcm sounds we use our own decoder
var storedContext = this;
for (var i=0; i<sounds.length; i++) {
@ -61,10 +67,12 @@ AudioEngine.prototype.loadSounds = function (sounds) {
var buffer = new Tone.Buffer();
this.audioBuffers[md5] = buffer;
// Squeak sound format (not implemented yet)
if (sounds[i].format == 'squeak') {
log.warn('unable to load sound in squeak format');
continue;
}
// most sounds decode natively, but for adpcm sounds we use our own decoder
if (sounds[i].format == 'adpcm') {
log.warn('loading sound in adpcm format');
// create a closure to store the sound md5, to use when the
@ -82,16 +90,33 @@ AudioEngine.prototype.loadSounds = function (sounds) {
}
};
/**
* Play a note for a duration on an instrument
* @param {number} note - a MIDI note number
* @param {number} beats - a duration in beats
* @param {number} inst - an instrument number (0-indexed)
* @return {Promise} a Promise that resolves after the duration has elapsed
*/
AudioEngine.prototype.playNoteForBeatsWithInst = function (note, beats, inst) {
var sec = this.beatsToSec(beats);
this.instrumentPlayer.playNoteForSecWithInst(note, sec, inst);
return this.waitForBeats(beats);
};
/**
* Convert a number of beats to a number of seconds, using the current tempo
* @param {number} beats
* @return {number} seconds
*/
AudioEngine.prototype.beatsToSec = function (beats) {
return (60 / this.currentTempo) * beats;
};
/**
* Wait for some number of beats
* @param {number} beats
* @return {Promise} a Promise that resolves after the duration has elapsed
*/
AudioEngine.prototype.waitForBeats = function (beats) {
var storedContext = this;
return new Promise(function (resolve) {
@ -101,26 +126,40 @@ AudioEngine.prototype.waitForBeats = function (beats) {
});
};
/**
* Set the global tempo in bpm (beats per minute)
* @param {number} value - the new tempo to set
*/
AudioEngine.prototype.setTempo = function (value) {
this.currentTempo = value;
};
/**
* Change the tempo by some number of bpm (beats per minute)
* @param {number} value - the number of bpm to change the tempo by
*/
AudioEngine.prototype.changeTempo = function (value) {
this.setTempo(this.currentTempo + value);
};
/**
* Create an AudioPlayer. Each sprite or clone has an AudioPlayer.
* It includes a reference to the AudioEngine so it can use global
* functionality such as playing notes.
* @return {AudioPlayer}
*/
AudioEngine.prototype.createPlayer = function () {
return new AudioPlayer(this);
};
/* Audio Player
Each sprite or clone has an audio player
the audio player handles sound playback, volume, and the sprite-specific audio effects:
pitch and pan
*/
/**
* Each sprite or clone has an audio player
* the audio player handles sound playback, volume, and the sprite-specific audio effects:
* pitch and pan
* @param {AudioEngine}
* @constructor
*/
function AudioPlayer (audioEngine) {
this.audioEngine = audioEngine;
@ -142,6 +181,11 @@ function AudioPlayer (audioEngine) {
this.activeSoundPlayers = Object.create(null);
}
/**
* Play a sound
* @param {string} md5 - the md5 id of a sound file
* @return {Promise} a Promise that resolves when the sound finishes playing
*/
AudioPlayer.prototype.playSound = function (md5) {
// if this sound is not in the audio engine, return
if (!this.audioEngine.audioBuffers[md5]) {
@ -173,11 +217,21 @@ AudioPlayer.prototype.playSound = function (md5) {
return player.finished();
};
/**
* Play a drum sound. The AudioEngine contains the DrumPlayer, but the AudioPlayer
* calls this function so that it can pass a reference to its own effects node.
* @param {number} drum - a drum number (0-indexed)
* @param {number} beats - a duration in beats
* @return {Promise} a Promise that resolves after the duration has elapsed
*/
AudioPlayer.prototype.playDrumForBeats = function (drum, beats) {
this.audioEngine.drumPlayer.play(drum, this.effectsNode);
return this.audioEngine.waitForBeats(beats);
};
/**
* Stop all sounds, notes and drums that are playing
*/
AudioPlayer.prototype.stopAllSounds = function () {
// stop all active sound players
for (var md5 in this.activeSoundPlayers) {
@ -191,6 +245,11 @@ AudioPlayer.prototype.stopAllSounds = function () {
this.audioEngine.drumPlayer.stopAll();
};
/**
* Set an audio effect to a value
* @param {string} effect - the name of the effect
* @param {number} value - the value to set the effect to
*/
AudioPlayer.prototype.setEffect = function (effect, value) {
switch (effect) {
case 'pitch':
@ -214,6 +273,9 @@ AudioPlayer.prototype.setEffect = function (effect, value) {
}
};
/**
* Clear all audio effects
*/
AudioPlayer.prototype.clearEffects = function () {
this.panEffect.set(0);
this.pitchEffect.set(0, this.activeSoundPlayers);
@ -225,6 +287,10 @@ AudioPlayer.prototype.clearEffects = function () {
this.audioEngine.roboticEffect.set(0);
};
/**
* Set the volume for sounds played by this AudioPlayer
* @param {number} value - the volume in range 0-100
*/
AudioPlayer.prototype.setVolume = function (value) {
this.effectsNode.gain.value = value / 100;
};