scratchfoundation/scratchjr
1
0
Fork 0
mirror of https://github.com/scratchfoundation/scratchjr.git synced 2024-11-28 18:15:37 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[WIP] new audio engine

Browse source 
Post coding session with Paula
This commit is contained in:
chrisgarrity 2017-02-21 09:05:59 -05:00
parent 57114ed678
commit 6a6d92e06b
8 changed files with 221 additions and 183 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

105
ios/ScratchJr/src/IO.m
View file

@ -3,6 +3,7 @@
ViewController* HTML;
MFMailComposeViewController *emailDialog;
NSMutableDictionary *mediastrings;
NSMutableDictionary *sounds;
// new primtives
@ -12,6 +13,7 @@ NSMutableDictionary *mediastrings;
// new primtives
+ (void)init:(ViewController*)vc {
mediastrings = [[NSMutableDictionary alloc] init];
sounds = [[NSMutableDictionary alloc] init];
HTML =vc;
}
@ -225,6 +227,109 @@ NSMutableDictionary *mediastrings;
return @"1";
}
////////////////////////////
// Sound System
////////////////////////////
+ (NSString *)registerSound:(NSString*)dir :(NSString*)name {
NSURL *url;
if ([dir isEqual: @"Documents"]){
url = [self getDocumentPath: name];
}
else {
url = [self getResourcePath: [NSString stringWithFormat: @"%@%@", dir, name]];
}
NSLog (@"registering %@", url);
NSError *error;
AVAudioPlayer *snd = [[AVAudioPlayer alloc] initWithContentsOfURL: url error:&error];
if (error == NULL) {
[sounds setObject:snd forKey:name];
return [NSString stringWithFormat: @"%@,%f", name, snd.duration];
}
else {
NSLog (@"%@", error);
}
return @"error";
}
// +(NSString *)playSound:(NSString*)name {
// // get the sound either from Documents (user defined sounds)
// // or from the HTML5 bundle.
// NSURL *url = ([dir isEqual: @"Documents"]) ? [self getDocumentPath: name] : [self getResou\
// rcePath: [NSString stringWithFormat: @"%@%@", dir, name]];
//
// // audio type: respect the "Mute" if there are audio sounds
// // ignore the Mute if it is from recording / playback and Runtime.
// NSString *audiotype = ([dir isEqual: @"Documents"] || [name isEqual:@"pop.mp3"]) ? AVAudio\
// SessionCategoryPlayAndRecord : AVAudioSessionCategoryAmbient;
// [[AVAudioSession sharedInstance] setCategory:audiotype error:nil];
//
// NSError *error;
// AVAudioPlayer *snd = [[AVAudioPlayer alloc] initWithContentsOfURL: url error:&error];
//
// if (error == NULL) {
// snd.numberOfLoops = 0;
// [snd prepareToPlay];
// [snd play];
// NSString *id = [self setSoundTimeout: snd];
// NSString *result = [NSString stringWithFormat: @"%@,%f", id, [snd duration]];
// NSLog (@"%@", result);
// return result;
// }
// else {
// NSLog (@"%@", error);
// return @"error";
// }
// }
+ (NSString *)playSound :(NSString*)name {
// TODO: make scratchJr pay attention to the mute
// // audio type: respect the "Mute" if there are audio sounds
// // ignore the Mute if it is from recording / playback and Runtime.
// NSString *audiotype = ([dir isEqual: @"Documents"] || [name isEqual:@"pop.mp3"]) ? AVAudio\
// SessionCategoryPlayAndRecord : AVAudioSessionCategoryAmbient;
// [[AVAudioSession sharedInstance] setCategory:audiotype error:nil];
AVAudioPlayer *snd = sounds[name];
NSLog (@"play %@", snd);
if (snd == NULL) {
return [NSString stringWithFormat: @"%@ not found", name];
}
else {
[snd prepareToPlay];
[snd play];
[NSTimer scheduledTimerWithTimeInterval: [snd duration] target: self selector: @selector(so\
undEnded:) userInfo:@{@"soundName": name} repeats: NO];
return [NSString stringWithFormat: @"%@ played", name];
}
}
+ (void)soundEnded:(NSTimer*)timer {
NSString *soundName = [[timer userInfo] objectForKey:@"soundName"];
NSLog(@"%@", soundName);
if (sounds [soundName] == NULL) return;
NSString *callback = [NSString stringWithFormat: @"iOS.soundDone('%@');",soundName];
UIWebView *webview = [ViewController webview];
[webview stringByEvaluatingJavaScriptFromString: callback];
}
+ (NSString *)stopSound :(NSString*)name {
AVAudioPlayer *snd = sounds[name];
NSLog (@"stop %@", snd);
if (snd == NULL) {
return [NSString stringWithFormat: @"%@ not found", name];
}
else {
[snd stop];
return [NSString stringWithFormat: @"%@ stopped", name];
}
}
////////////////////////////
// File system
////////////////////////////

8
ios/ScratchJr/src/ScratchJr.h
View file

@ -103,6 +103,10 @@
-(NSString*) io_getmedialen:(NSString*)file :(NSString*)key;
-(NSString*) io_getmediadone:(NSString*)filename;
-(NSString*) io_remove:(NSString*)filename;
-(NSString*) io_registersound:(NSString*) dir :(NSString*) name;
-(NSString*) io_playsound:(NSString*) name;
-(NSString*) io_stopsound:(NSString*) name;
-(NSString*) recordsound_recordstart;
-(NSString*) recordsound_recordstop;
-(NSString*) recordsound_volume;
@ -158,6 +162,9 @@
+ (NSString*) getmediadone:(NSString*)filename;
+ (NSString*) remove:(NSString*)filename;
+ (NSString*) sendSjrUsingShareDialog:(NSString*) fileName :(NSString*) emailSubject :(NSString*) emailBody :(int) shareType :(NSString*) b64data;
+(NSString *) registerSound:(NSString*) dir :(NSString*) name;
+(NSString *) playSound:(NSString*) name;
+(NSString *) stopSound:(NSString*) name;
@end
@interface ScratchJr : NSObject
@ -175,4 +182,3 @@
+(NSString*) choosecamera:(NSString*) body;
+(NSString*) captureimage:(NSString*)onCameraCaptureComplete;
@end

12
ios/ScratchJr/src/ViewController.m
View file

@ -256,6 +256,18 @@ JSContext *js;
return [IO remove:filename];
}
-(NSString*) io_registersound:(NSString*)dir :(NSString*)name {
return [IO registerSound:dir:name];
}
-(NSString*) io_playsound:(NSString*) name {
return [IO playSound:name];
}
-(NSString*) io_stopsound:(NSString*) name {
return [IO stopSound:name];
}
-(NSString*) recordsound_recordstart {
return [RecordSound startRecord];
}

17
src/editor/ScratchJr.js
View file

@ -229,7 +229,7 @@ export default class ScratchJr {
document.ontouchmove = function (e) {
e.preventDefault();
};
window.ontouchstart = ScratchJr.triggerAudio;
window.ontouchstart = ScratchJr.unfocus;
if (isTablet) {
window.ontouchend = undefined;
} else {
@ -237,20 +237,6 @@ export default class ScratchJr {
}
}
static prepareAudio () {
if (ScratchAudio.firstTime) {
ScratchAudio.firstClick();
}
if (!ScratchAudio.firstTime) {
window.ontouchstart = ScratchJr.unfocus;
}
}
static triggerAudio (evt) {
ScratchJr.prepareAudio();
ScratchJr.unfocus(evt);
}
static unfocus (evt) {
if (Palette.helpballoon) {
Palette.helpballoon.parentNode.removeChild(Palette.helpballoon);
@ -456,7 +442,6 @@ export default class ScratchJr {
}
static runStrips (e) {
ScratchJr.prepareAudio();
ScratchJr.stopStripsFromTop(e);
ScratchJr.unfocus(e);
ScratchJr.startGreenFlagThreads();

21
src/editor/ui/Record.js
View file

@ -289,7 +289,7 @@ export default class Record {
}
static closeContinueSave () {
iOS.recorddisappear('YES', Record.getUserSound);
iOS.recorddisappear('YES', Record.registerProjectSound);
}
static closeContinueRemove () {
@ -297,18 +297,8 @@ export default class Record {
iOS.recorddisappear('NO', Record.tearDownRecorder);
}
static getUserSound () {
isRecording = false;
if (!isAndroid) {
iOS.getmedia(recordedSound, Record.registerProjectSound);
} else {
// On Android, just pass URL
Record.registerProjectSound(null);
}
}
static registerProjectSound (data) {
function loadingDone (snd) {
static registerProjectSound () {
function whenDone (snd) {
if (snd != 'error') {
var spr = ScratchJr.getSprite();
var page = spr.div.parentNode.owner;
@ -325,10 +315,10 @@ export default class Record {
Palette.selectCategory(3);
}
if (!isAndroid) {
ScratchAudio.loadFromData(recordedSound, data, loadingDone);
ScratchAudio.loadFromLocal('Documents', recordedSound, whenDone);
} else {
// On Android, just pass URL
ScratchAudio.loadFromLocal(recordedSound, loadingDone);
ScratchAudio.loadFromLocal('', recordedSound, whenDone);
}
}
@ -352,7 +342,6 @@ export default class Record {
error = false;
}
// Refresh audio context
ScratchAudio.firstTime = true;
isRecording = false;
recordedSound = null;
// Hide the dialog

37
src/iPad/iOS.js
View file

@ -2,6 +2,7 @@ import {isiOS, gn} from '../utils/lib';
import IO from './IO';
import Lobby from '../lobby/Lobby';
import Alert from '../editor/ui/Alert';
import ScratchAudio from '../utils/ScratchAudio';
//////////////////////////////////////////////////
// Tablet interface functions
@ -180,6 +181,40 @@ export default class iOS {
}
}
// Sound functions
static registerSound (dir, name, fcn) {
var result = tabletInterface.io_registersound(dir, name);
console.log(result);
if (fcn) {
fcn(result);
}
}
static playSound (name, fcn) {
var result = tabletInterface.io_playsound(name);
console.log(result); // eslint-disable-line no-console
if (fcn) {
fcn(result);
}
}
static stopSound (name, fcn) {
var result = tabletInterface.io_stopsound(name);
if (fcn) {
fcn(result);
}
}
// Web Wiew delegate call backs
static soundDone (name) {
console.log('soundDone callback, id:', name); // eslint-disable-line no-console
ScratchAudio.soundDone(name);
}
static sndrecord (fcn) {
var result = tabletInterface.recordsound_recordstart();
if (fcn) {
@ -341,6 +376,8 @@ export default class iOS {
}
}
}
}
// Expose iOS methods for ScratchJr tablet sharing callbacks

155
src/utils/ScratchAudio.js
View file

@ -7,50 +7,19 @@ import iOS from '../iPad/iOS';
////////////////////////////////////////////////////
let uiSounds = {};
let context;
let firstTime = true;
let defaultSounds = ['cut.wav', 'snap.wav', 'copy.wav', 'grab.wav', 'boing.wav', 'tap.wav',
'keydown.wav', 'entertap.wav', 'exittap.wav', 'splash.wav'];
let projectSounds = {};
let path = '';
export default class ScratchAudio {
static get uiSounds () {
return uiSounds;
}
static get firstTime () {
return firstTime;
}
static set firstTime (newFirstTime) {
firstTime = newFirstTime;
}
static get projectSounds () {
return projectSounds;
}
static get context () {
return context;
}
static firstClick () { // trick to abilitate the Audio context in iOS 8+
var res = true;
if (uiSounds['keydown.wav']) {
uiSounds['keydown.wav'].playWithVolume(0);
res = false;
}
firstTime = res;
}
static firstOnTouchEnd () { // trick to abilitate the Audio context in iOS 9
if (uiSounds['keydown.wav']) {
uiSounds['keydown.wav'].playWithVolume(0);
}
window.removeEventListener('touchend', ScratchAudio.firstOnTouchEnd, false);
}
static sndFX (name) {
ScratchAudio.sndFXWithVolume(name, 1.0);
}
@ -60,8 +29,7 @@ export default class ScratchAudio {
if (!uiSounds[name]) {
return;
}
uiSounds[name].playWithVolume(volume);
firstTime = false;
uiSounds[name].play();
} else {
AndroidInterface.audio_sndfxwithvolume(name, volume);
}
@ -72,49 +40,41 @@ export default class ScratchAudio {
prefix = '';
}
if (!isAndroid) {
context = new webkitAudioContext();
} else {
context = {
decodeAudioData: function () {
},
play: function () {
}
};
prefix = 'HTML5/';
}
uiSounds = {};
for (var i = 0; i < defaultSounds.length; i++) {
ScratchAudio.addSound(prefix + 'sounds/', defaultSounds[i], uiSounds);
}
ScratchAudio.addSound(path, prefix + 'pop.mp3', projectSounds);
ScratchAudio.addSound(prefix, 'pop.mp3', projectSounds);
// } else {
// for (var j=0; j < defaultSounds.length; j++) {
// iOS.registerSound('HTML5/sounds/', defaultSounds[j], ScratchAudio.UIsoundLoaded );
// }
// iOS.registerSound('HTML5/', 'pop.mp3', ScratchAudio.UIsoundLoaded );
// }
}
static addSound (url, snd, dict, fcn) {
var name = snd;
console.log(url+' '+snd);
if (!isAndroid) {
var whenDone = function (str) {
if (str != 'error') {
var result = snd.split (',');
dict[snd] = new Sound(result[0], result[1]);
} else {
name = 'error';
}
if (fcn) {
fcn(name);
}
// dict [name].time = Number (result[1]);
};
iOS.registerSound(url, snd, whenDone);
var bufferSound = function () {
context.decodeAudioData(request.response, onDecode, onDecodeError);
};
var onDecodeError = function () {
if (fcn) {
fcn('error');
}
};
var onDecode = function (buffer) {
dict[snd] = new Sound(buffer);
if (fcn) {
fcn(snd);
}
};
var transferFailed = function (e) {
e.preventDefault();
e.stopPropagation();
};
var request = new XMLHttpRequest();
request.open('GET', url + snd, true);
request.responseType = 'arraybuffer';
request.addEventListener('load', bufferSound, false);
request.addEventListener('error', transferFailed, false);
request.send(null);
} else {
// In Android, this is handled outside of JavaScript, so just place a stub here.
dict[snd] = new Sound(url + snd);
@ -124,65 +84,34 @@ export default class ScratchAudio {
}
}
static soundDone (name) {
if (!projectSounds[name]) return;
projectSounds[name].playing = false;
console.log(name);
}
static loadProjectSound (md5, fcn) {
console.log(md5);
if (!md5) {
return;
}
if (md5.indexOf('/') > -1) {
ScratchAudio.loadFromLocal(md5, fcn);
} else {
if (md5.indexOf('wav') > -1) {
if (!isAndroid) {
iOS.getmedia(md5, nextStep);
} else {
// On Android, all sounds play server-side
ScratchAudio.loadFromLocal(md5, fcn);
}
} else {
ScratchAudio.loadFromLocal(md5, fcn);
}
}
function nextStep (data) {
ScratchAudio.loadFromData(md5, data, fcn);
var dir = '';
if (!isAndroid) {
if (md5.indexOf('/') > -1) dir = 'HTML5/';
else if (md5.indexOf('wav') > -1) dir = 'Documents';
}
console.log('loadProjectSound: ' + dir + ' ' + md5);
ScratchAudio.loadFromLocal(dir, md5, fcn);
}
static loadFromLocal (md5, fcn) {
static loadFromLocal (dir, md5, fcn) {
if (projectSounds[md5] != undefined) {
return;
}
ScratchAudio.addSound(path, md5, projectSounds, fcn);
ScratchAudio.addSound(dir, md5, projectSounds, fcn);
}
static loadFromData (md5, data, fcn) {
if (!data) {
projectSounds[md5] = projectSounds['pop.mp3'];
} else {
var onDecode = function (buffer) {
projectSounds[md5] = new Sound(buffer);
if (fcn) {
fcn(md5);
}
};
var onError = function () {
// console.log ("error", md5, err);
if (fcn) {
fcn('error');
}
};
var byteString = atob(data); // take out the base 64 encoding
var buffer = new ArrayBuffer(byteString.length);
var bytearray = new Uint8Array(buffer);
for (var i = 0; i < byteString.length; i++) {
bytearray[i] = byteString.charCodeAt(i);
}
context.decodeAudioData(buffer, onDecode, onError);
}
}
}
window.ScratchAudio = ScratchAudio;
window.addEventListener('touchend', ScratchAudio.firstOnTouchEnd, false);

49
src/utils/Sound.js
View file

@ -2,13 +2,14 @@ import {isAndroid} from './lib';
import ScratchAudio from './ScratchAudio';
export default class Sound {
constructor (buffer) {
constructor (name, time) {
if (isAndroid) {
this.url = buffer;
this.url = name;
this.soundPlayId = null;
} else {
this.buffer = buffer;
this.source = null;
this.name = name;
this.time = time;
this.playing = false;
}
}
@ -19,37 +20,11 @@ export default class Sound {
}
this.soundPlayId = AndroidInterface.audio_play(this.url, 1.0);
} else {
if (this.source) {
if (this.playing) {
this.stop();
}
this.source = ScratchAudio.context.createBufferSource();
this.source.buffer = this.buffer;
this.source.connect(ScratchAudio.context.destination);
this.source.noteOn(0);
}
}
playWithVolume (n) {
if (isAndroid) {
if (this.soundPlayId) {
this.stop();
}
if (n > 0) {
// This method is not currently called with any value other than 0. If 0, don't play the sound.
this.soundPlayId = AndroidInterface.audio_play(this.url, n);
}
} else {
if (this.source) {
this.stop();
}
this.gainNode = ScratchAudio.context.createGainNode();
this.source = ScratchAudio.context.createBufferSource();
this.source.buffer = this.buffer;
this.source.connect(this.gainNode);
this.gainNode.connect(ScratchAudio.context.destination);
this.source.noteOn(0);
this.gainNode.gain.value = n;
iOS.playSound(this.name);
this.playing = true;
}
}
@ -57,7 +32,7 @@ export default class Sound {
if (isAndroid) {
return (this.soundPlayId == null) || !AndroidInterface.audio_isplaying(this.soundPlayId);
} else {
return (this.source == null) || (this.source.playbackState == 3);
return (!this.playing);
}
}
@ -65,7 +40,7 @@ export default class Sound {
if (isAndroid) {
this.soundPlayId = null;
} else {
this.source = null;
this.playing = false;
}
}
@ -76,8 +51,8 @@ export default class Sound {
}
this.soundPlayId = null;
} else {
this.source.noteOff(0);
this.source = null;
iOS.stopSound(this.name);
this.playing = false;
}
}
}