Merge pull request #121 from chrisgarrity/bug/iOS-sound

Rewrite iOS sound management to use native audio instead of webaudio. Quirks are consistent with Android which was already doing native audio.

editions/free/src/app.bundle.js.map

@@ -0,0 +1 @@
+../../../src/build/bundles/app.bundle.js.map

ios/ScratchJr/src/IO.m

@@ -3,6 +3,8 @@
 ViewController* HTML;
 MFMailComposeViewController *emailDialog;
 NSMutableDictionary *mediastrings;
+NSMutableDictionary *sounds;
+NSMutableDictionary *soundtimers;
 
 // new primtives
 
@@ -12,6 +14,8 @@ NSMutableDictionary *mediastrings;
 // new primtives
 + (void)init:(ViewController*)vc {
     mediastrings = [[NSMutableDictionary alloc] init];
+    sounds = [[NSMutableDictionary alloc] init];
+    soundtimers = [[NSMutableDictionary alloc] init];
     HTML =vc;
 }
 
@@ -225,6 +229,73 @@ NSMutableDictionary *mediastrings;
     return @"1";
 }
 
+////////////////////////////
+// Sound System
+////////////////////////////
+
++ (NSString *)registerSound:(NSString*)dir :(NSString*)name  {
+    NSURL *url;
+    if ([dir isEqual:@"Documents"]){
+        url = [self getDocumentPath: name];
+    }
+    else {
+        url = [self getResourcePath: [NSString stringWithFormat: @"%@%@", dir, name]];
+    }
+
+    NSError *error;
+    AVAudioPlayer *snd = [[AVAudioPlayer alloc] initWithContentsOfURL: url error:&error];
+
+    if (error == nil) {
+        [sounds setObject:snd forKey:name];
+        [snd prepareToPlay];
+        return [NSString stringWithFormat: @"%@,%f", name,  snd.duration];
+    }
+    return @"error";
+}
+
++ (NSString *)playSound :(NSString*)name  {
+    // TODO: make scratchJr pay attention to the mute
+    //         // audio type: respect the "Mute" if there are audio sounds
+    //         // ignore the Mute if it is from recording / playback and Runtime.
+    //         NSString *audiotype = ([dir  isEqual: @"Documents"] || [name isEqual:@"pop.mp3"]) ? AVAudio\
+    // SessionCategoryPlayAndRecord : AVAudioSessionCategoryAmbient;
+    //         [[AVAudioSession sharedInstance] setCategory:audiotype error:nil];
+    AVAudioPlayer *snd = sounds[name];
+    if (snd == nil) {
+        return [NSString stringWithFormat:@"%@ not found", name];
+    }
+    NSTimer *sndTimer = soundtimers[name];
+    if (sndTimer.valid) {
+        // this sound is already playing, invalidate so that new timer will overrule
+        [sndTimer invalidate];
+    }
+    [snd setCurrentTime:0];
+    [snd play];
+    [soundtimers setObject:[NSTimer scheduledTimerWithTimeInterval:[snd duration]
+                                     target:self
+                                   selector:@selector(soundEnded:)
+                                   userInfo:@{@"soundName":name}
+                                    repeats:NO] forKey:name];
+    return  [NSString stringWithFormat:@"%@ played", name];
+}
+
++ (void)soundEnded:(NSTimer*)timer {
+        NSString *soundName = [[timer userInfo] objectForKey:@"soundName"];
+        if (sounds[soundName] == nil) return;
+        NSString *callback = [NSString stringWithFormat:@"iOS.soundDone('%@');", soundName];
+        UIWebView *webview = [ViewController webview];
+        [webview stringByEvaluatingJavaScriptFromString:callback];
+    }
+
++ (NSString *)stopSound :(NSString*)name  {
+    AVAudioPlayer *snd = sounds[name];
+    if (snd == nil) {
+        return [NSString stringWithFormat:@"%@ not found", name];
+    }
+    [snd stop];
+    return  [NSString stringWithFormat:@"%@ stopped", name];
+}
+
 ////////////////////////////
 // File system
 ////////////////////////////

ios/ScratchJr/src/ScratchJr.h

@@ -103,6 +103,10 @@
 - (NSString *)io_getmedialen:(NSString *)file :(NSString *)key;
 - (NSString *)io_getmediadone:(NSString *)filename;
 - (NSString *)io_remove:(NSString *)filename;
+- (NSString *)io_registersound:(NSString *)dir :(NSString *)name;
+- (NSString *)io_playsound:(NSString *)name;
+- (NSString *)io_stopsound:(NSString *)name;
+
 - (NSString *)recordsound_recordstart;
 - (NSString *)recordsound_recordstop;
 - (NSString *)recordsound_volume;
@@ -116,7 +120,11 @@
 - (NSString *)scratchjr_stopfeed;
 - (NSString *)scratchjr_choosecamera:(NSString *)body;
 - (NSString *)scratchjr_captureimage:(NSString *)onCameraCaptureComplete;
-- (NSString*) sendSjrUsingShareDialog:(NSString*) fileName :(NSString*) emailSubject :(NSString*) emailBody :(int) shareType :(NSString*) b64data;
+- (NSString *)sendSjrUsingShareDialog:(NSString *)fileName
+                                     :(NSString *)emailSubject 
+                                     :(NSString *)emailBody
+                                     :(int)shareType
+                                     :(NSString *)b64data;
 - (NSString *) deviceName;
 - (NSString *) analyticsEvent:(NSString *)category :(NSString *)action :(NSString *)label :(NSNumber*)value;
 @end
@@ -131,7 +139,10 @@
 @end
 
 @interface ViewController (ViewFinderDelegate) <ViewFinderDelegate>
-- (void) showShareEmail:(NSURL *) projectURL withName: (NSString*) name withSubject:(NSString*) subject withBody:(NSString*)body;
+- (void)showShareEmail:(NSURL *)projectURL
+              withName:(NSString *)name
+           withSubject:(NSString *)subject
+              withBody:(NSString *)body;
 - (void)showShareAirdrop:(NSURL *)projectURL;
 @end
 
@@ -157,7 +168,14 @@
 + (NSString *)getmedialen:(NSString *)file :(NSString *)key;
 + (NSString *)getmediadone:(NSString *)filename;
 + (NSString *)remove:(NSString *)filename;
-+ (NSString*) sendSjrUsingShareDialog:(NSString*) fileName :(NSString*) emailSubject :(NSString*) emailBody :(int) shareType :(NSString*) b64data;
++ (NSString *)sendSjrUsingShareDialog:(NSString *)fileName
+                                     :(NSString *)emailSubject
+                                     :(NSString *)emailBody
+                                     :(int)shareType
+                                     :(NSString *)b64data;
++ (NSString *)registerSound:(NSString *)dir :(NSString *)name;
++ (NSString *)playSound:(NSString *)name;
++ (NSString *)stopSound:(NSString *)name;
 @end
 
 @interface ScratchJr : NSObject
@@ -175,4 +193,3 @@
 + (NSString *)choosecamera:(NSString *)body;
 + (NSString *)captureimage:(NSString *)onCameraCaptureComplete;
 @end
-

ios/ScratchJr/src/ViewController.m

@@ -78,7 +78,6 @@ JSContext *js;
             [defaultsToRegister setObject:[prefSpecification objectForKey:@"DefaultValue"] forKey:key];
         }
     }
-  //  NSLog(@"defaultsToRegister %@", defaultsToRegister);
     [[NSUserDefaults standardUserDefaults] registerDefaults:defaultsToRegister];
 }
 
@@ -256,6 +255,18 @@ JSContext *js;
     return [IO remove:filename];
 }
 
+-(NSString*) io_registersound:(NSString*)dir :(NSString*)name {
+    return [IO registerSound:dir:name];
+}
+
+-(NSString*) io_playsound:(NSString*) name {
+    return [IO playSound:name];
+}
+
+-(NSString*) io_stopsound:(NSString*) name {
+    return [IO stopSound:name];
+}
+
 -(NSString*) recordsound_recordstart {
     return [RecordSound startRecord];
 }

src/editor/ScratchJr.js

@@ -229,7 +229,7 @@ export default class ScratchJr {
         document.ontouchmove = function (e) {
             e.preventDefault();
         };
-        window.ontouchstart = ScratchJr.triggerAudio;
+        window.ontouchstart = ScratchJr.unfocus;
         if (isTablet) {
             window.ontouchend = undefined;
         } else {
@@ -237,20 +237,6 @@ export default class ScratchJr {
         }
     }
 
-    static prepareAudio () {
-        if (ScratchAudio.firstTime) {
-            ScratchAudio.firstClick();
-        }
-        if (!ScratchAudio.firstTime) {
-            window.ontouchstart = ScratchJr.unfocus;
-        }
-    }
-
-    static triggerAudio (evt) {
-        ScratchJr.prepareAudio();
-        ScratchJr.unfocus(evt);
-    }
-
     static unfocus (evt) {
         if (Palette.helpballoon) {
             Palette.helpballoon.parentNode.removeChild(Palette.helpballoon);
@@ -456,7 +442,6 @@ export default class ScratchJr {
     }
 
     static runStrips (e) {
-        ScratchJr.prepareAudio();
         ScratchJr.stopStripsFromTop(e);
         ScratchJr.unfocus(e);
         ScratchJr.startGreenFlagThreads();

src/editor/ui/Record.js

@@ -289,7 +289,7 @@ export default class Record {
     }
 
     static closeContinueSave () {
-        iOS.recorddisappear('YES', Record.getUserSound);
+        iOS.recorddisappear('YES', Record.registerProjectSound);
     }
 
     static closeContinueRemove () {
@@ -297,18 +297,8 @@ export default class Record {
         iOS.recorddisappear('NO', Record.tearDownRecorder);
     }
 
-    static getUserSound () {
+    static registerProjectSound () {
-        isRecording = false;
+        function whenDone (snd) {
-        if (!isAndroid) {
-            iOS.getmedia(recordedSound, Record.registerProjectSound);
-        } else {
-            // On Android, just pass URL
-            Record.registerProjectSound(null);
-        }
-    }
-
-    static registerProjectSound (data) {
-        function loadingDone (snd) {
             if (snd != 'error') {
                 var spr = ScratchJr.getSprite();
                 var page = spr.div.parentNode.owner;
@@ -325,10 +315,10 @@ export default class Record {
             Palette.selectCategory(3);
         }
         if (!isAndroid) {
-            ScratchAudio.loadFromData(recordedSound, data, loadingDone);
+            ScratchAudio.loadFromLocal('Documents', recordedSound, whenDone);
         } else {
             // On Android, just pass URL
-            ScratchAudio.loadFromLocal(recordedSound, loadingDone);
+            ScratchAudio.loadFromLocal('', recordedSound, whenDone);
         }
     }
 
@@ -352,7 +342,6 @@ export default class Record {
             error = false;
         }
         // Refresh audio context
-        ScratchAudio.firstTime = true;
         isRecording = false;
         recordedSound = null;
         // Hide the dialog

src/entry/index.js

@@ -53,7 +53,6 @@ function indexFirstTime () {
         iOS.hidesplash(doit);
     }, 500);
     function doit () {
-        ScratchAudio.sndFX('tap.wav');
         window.ontouchend = function () {
             indexLoadOptions();
         };
@@ -88,13 +87,6 @@ function indexLoadOptions () {
 }
 
 function indexGohome () {
-    // On iOS, sounds are loaded async, but the code as written expects to play tap.wav when we enter home.html
-    // (but since it isn't loaded yet, no sound is played).
-    // On Android, sync sounds means both calls to tap.wav result in a sound play.
-    // XXX: we should re-write the lobby loading to wait for the sounds to load, and not play a sound here.
-    if (isiOS) {
-        ScratchAudio.sndFX('tap.wav');
-    }
     iOS.setfile('homescroll.sjr', 0, function () {
         doNext();
     });

src/iPad/iOS.js

@@ -2,6 +2,7 @@ import {isiOS, gn} from '../utils/lib';
 import IO from './IO';
 import Lobby from '../lobby/Lobby';
 import Alert from '../editor/ui/Alert';
+import ScratchAudio from '../utils/ScratchAudio';
 
 //////////////////////////////////////////////////
 //  Tablet interface functions
@@ -180,6 +181,35 @@ export default class iOS {
         }
     }
 
+    // Sound functions
+
+    static registerSound (dir, name, fcn) {
+        var result = tabletInterface.io_registersound(dir, name);
+        if (fcn) {
+            fcn(result);
+        }
+    }
+
+    static playSound (name, fcn) {
+        var result = tabletInterface.io_playsound(name);
+        if (fcn) {
+            fcn(result);
+        }
+    }
+
+    static stopSound (name, fcn) {
+        var result = tabletInterface.io_stopsound(name);
+        if (fcn) {
+            fcn(result);
+        }
+    }
+
+    // Web Wiew delegate call backs
+
+    static soundDone (name) {
+        ScratchAudio.soundDone(name);
+    }
+
     static sndrecord (fcn) {
         var result = tabletInterface.recordsound_recordstart();
         if (fcn) {

src/utils/ScratchAudio.js

@@ -7,50 +7,19 @@ import iOS from '../iPad/iOS';
 ////////////////////////////////////////////////////
 
 let uiSounds = {};
-let context;
-let firstTime = true;
 let defaultSounds = ['cut.wav', 'snap.wav', 'copy.wav', 'grab.wav', 'boing.wav', 'tap.wav',
     'keydown.wav', 'entertap.wav', 'exittap.wav', 'splash.wav'];
 let projectSounds = {};
-let path = '';
 
 export default class ScratchAudio {
     static get uiSounds () {
         return uiSounds;
     }
 
-    static get firstTime () {
-        return firstTime;
-    }
-
-    static set firstTime (newFirstTime) {
-        firstTime = newFirstTime;
-    }
-
     static get projectSounds () {
         return projectSounds;
     }
 
-    static get context () {
-        return context;
-    }
-
-    static firstClick () { // trick to abilitate the Audio context in iOS 8+
-        var res = true;
-        if (uiSounds['keydown.wav']) {
-            uiSounds['keydown.wav'].playWithVolume(0);
-            res = false;
-        }
-        firstTime = res;
-    }
-
-    static firstOnTouchEnd () { // trick to abilitate the Audio context in iOS 9
-        if (uiSounds['keydown.wav']) {
-            uiSounds['keydown.wav'].playWithVolume(0);
-        }
-        window.removeEventListener('touchend', ScratchAudio.firstOnTouchEnd, false);
-    }
-
     static sndFX (name) {
         ScratchAudio.sndFXWithVolume(name, 1.0);
     }
@@ -60,8 +29,7 @@ export default class ScratchAudio {
             if (!uiSounds[name]) {
                 return;
             }
-            uiSounds[name].playWithVolume(volume);
+            uiSounds[name].play();
-            firstTime = false;
         } else {
             AndroidInterface.audio_sndfxwithvolume(name, volume);
         }
@@ -72,49 +40,31 @@ export default class ScratchAudio {
             prefix = '';
         }
         if (!isAndroid) {
-            context = new webkitAudioContext();
+            prefix = 'HTML5/';
-        } else {
-            context = {
-                decodeAudioData: function () {
-                },
-                play: function () {
-                }
-            };
         }
         uiSounds = {};
+
         for (var i = 0; i < defaultSounds.length; i++) {
             ScratchAudio.addSound(prefix + 'sounds/', defaultSounds[i], uiSounds);
         }
-        ScratchAudio.addSound(path, prefix + 'pop.mp3', projectSounds);
+        ScratchAudio.addSound(prefix, 'pop.mp3', projectSounds);
     }
 
     static addSound (url, snd, dict, fcn) {
+        var name = snd;
         if (!isAndroid) {
-
+            var whenDone =  function (str) {
-            var bufferSound = function () {
+                if (str != 'error') {
-                context.decodeAudioData(request.response, onDecode, onDecodeError);
+                    var result = snd.split (',');
-            };
+                    dict[snd] = new Sound(result[0], result[1]);
-            var onDecodeError = function () {
+                } else {
+                    name = 'error';
+                }
                 if (fcn) {
-                    fcn('error');
+                    fcn(name);
                 }
             };
-            var onDecode = function (buffer) {
+            iOS.registerSound(url, snd, whenDone);
-                dict[snd] = new Sound(buffer);
-                if (fcn) {
-                    fcn(snd);
-                }
-            };
-            var transferFailed = function (e) {
-                e.preventDefault();
-                e.stopPropagation();
-            };
-            var request = new XMLHttpRequest();
-            request.open('GET', url + snd, true);
-            request.responseType = 'arraybuffer';
-            request.addEventListener('load', bufferSound, false);
-            request.addEventListener('error', transferFailed, false);
-            request.send(null);
         } else {
             // In Android, this is handled outside of JavaScript, so just place a stub here.
             dict[snd] = new Sound(url + snd);
@@ -124,65 +74,29 @@ export default class ScratchAudio {
         }
     }
 
+    static soundDone (name) {
+        if (!projectSounds[name]) return;
+        projectSounds[name].playing = false;
+    }
+
     static loadProjectSound (md5, fcn) {
         if (!md5) {
             return;
         }
-        if (md5.indexOf('/') > -1) {
+        var dir = '';
-            ScratchAudio.loadFromLocal(md5, fcn);
-        } else {
-
-            if (md5.indexOf('wav') > -1) {
         if (!isAndroid) {
-                    iOS.getmedia(md5, nextStep);
+            if (md5.indexOf('/') > -1) dir = 'HTML5/';
-                } else {
+            else if (md5.indexOf('wav') > -1) dir = 'Documents';
-                    // On Android, all sounds play server-side
-                    ScratchAudio.loadFromLocal(md5, fcn);
-                }
-            } else {
-                ScratchAudio.loadFromLocal(md5, fcn);
-            }
-        }
-        function nextStep (data) {
-            ScratchAudio.loadFromData(md5, data, fcn);
         }
+        ScratchAudio.loadFromLocal(dir, md5, fcn);
     }
 
-    static loadFromLocal (md5, fcn) {
+    static loadFromLocal (dir, md5, fcn) {
         if (projectSounds[md5] != undefined) {
             return;
         }
-        ScratchAudio.addSound(path, md5, projectSounds, fcn);
+        ScratchAudio.addSound(dir, md5, projectSounds, fcn);
-    }
-
-    static loadFromData (md5, data, fcn) {
-        if (!data) {
-            projectSounds[md5] = projectSounds['pop.mp3'];
-        } else {
-            var onDecode = function (buffer) {
-                projectSounds[md5] = new Sound(buffer);
-                if (fcn) {
-                    fcn(md5);
-                }
-            };
-            var onError = function () {
-                //	console.log ("error", md5, err);
-                if (fcn) {
-                    fcn('error');
-                }
-            };
-            var byteString = atob(data); // take out the base 64 encoding
-            var buffer = new ArrayBuffer(byteString.length);
-            var bytearray = new Uint8Array(buffer);
-            for (var i = 0; i < byteString.length; i++) {
-                bytearray[i] = byteString.charCodeAt(i);
-            }
-            context.decodeAudioData(buffer, onDecode, onError);
-
-        }
     }
 }
 
 window.ScratchAudio = ScratchAudio;
-
-window.addEventListener('touchend', ScratchAudio.firstOnTouchEnd, false);

src/utils/Sound.js

@@ -1,14 +1,15 @@
 import {isAndroid} from './lib';
-import ScratchAudio from './ScratchAudio';
+import iOS from '../iPad/iOS';
 
 export default class Sound {
-    constructor (buffer) {
+    constructor (name, time) {
         if (isAndroid) {
-            this.url = buffer;
+            this.url = name;
             this.soundPlayId = null;
         } else {
-            this.buffer = buffer;
+            this.name = name;
-            this.source = null;
+            this.time = time;
+            this.playing = false;
         }
     }
 
@@ -19,37 +20,11 @@ export default class Sound {
             }
             this.soundPlayId = AndroidInterface.audio_play(this.url, 1.0);
         } else {
-            if (this.source) {
+            if (this.playing) {
                 this.stop();
             }
-            this.source = ScratchAudio.context.createBufferSource();
+            iOS.playSound(this.name);
-            this.source.buffer = this.buffer;
+            this.playing = true;
-            this.source.connect(ScratchAudio.context.destination);
-            this.source.noteOn(0);
-        }
-    }
-
-    playWithVolume (n) {
-        if (isAndroid) {
-            if (this.soundPlayId) {
-                this.stop();
-            }
-
-            if (n > 0) {
-                // This method is not currently called with any value other than 0. If 0, don't play the sound.
-                this.soundPlayId = AndroidInterface.audio_play(this.url, n);
-            }
-        } else {
-            if (this.source) {
-                this.stop();
-            }
-            this.gainNode = ScratchAudio.context.createGainNode();
-            this.source = ScratchAudio.context.createBufferSource();
-            this.source.buffer = this.buffer;
-            this.source.connect(this.gainNode);
-            this.gainNode.connect(ScratchAudio.context.destination);
-            this.source.noteOn(0);
-            this.gainNode.gain.value = n;
         }
     }
 
@@ -57,7 +32,7 @@ export default class Sound {
         if (isAndroid) {
             return (this.soundPlayId == null) || !AndroidInterface.audio_isplaying(this.soundPlayId);
         } else {
-            return (this.source == null) || (this.source.playbackState == 3);
+            return (!this.playing);
         }
     }
 
@@ -65,7 +40,7 @@ export default class Sound {
         if (isAndroid) {
             this.soundPlayId = null;
         } else {
-            this.source = null;
+            this.playing = false;
         }
     }
 
@@ -76,8 +51,8 @@ export default class Sound {
             }
             this.soundPlayId = null;
         } else {
-            this.source.noteOff(0);
+            iOS.stopSound(this.name);
-            this.source = null;
+            this.playing = false;
         }
     }
 }