Merge pull request #979 from kchadha/serialization-cleanup

SB3 Serialization & Load Project

src/serialization/sb2.js

@@ -224,9 +224,16 @@ const parseScratchObject = function (object, runtime, extensions, topLevel) {
                 bitmapResolution: costumeSource.bitmapResolution || 1,
                 rotationCenterX: costumeSource.rotationCenterX,
                 rotationCenterY: costumeSource.rotationCenterY,
+                // TODO we eventually want this next property to be called
+                // md5ext to reflect what it actually contains, however this
+                // will be a very extensive change across many repositories
+                // and should be done carefully and altogether
+                md5: costumeSource.baseLayerMD5,
                 skinId: null
             };
-            costumePromises.push(loadCostume(costumeSource.baseLayerMD5, costume, runtime));
+            // TODO need to add deserializeCostume here so that assets from
+            // actual .sb2s get loaded in
+            costumePromises.push(loadCostume(costume.md5, costume, runtime));
         }
     }
     // Sounds from JSON
@@ -240,9 +247,17 @@ const parseScratchObject = function (object, runtime, extensions, topLevel) {
                 rate: soundSource.rate,
                 sampleCount: soundSource.sampleCount,
                 soundID: soundSource.soundID,
+                // TODO we eventually want this next property to be called
+                // md5ext to reflect what it actually contains, however this
+                // will be a very extensive change across many repositories
+                // and should be done carefully and altogether
+                // (for example, the audio engine currently relies on this
+                // property to be named 'md5')
                 md5: soundSource.md5,
                 data: null
             };
+            // TODO need to add deserializeSound here so that assets from
+            // actual .sb2s get loaded in
             soundPromises.push(loadSound(sound, runtime));
         }
     }

src/serialization/sb3.js

@@ -25,6 +25,95 @@ const {deserializeCostume, deserializeSound} = require('./deserialize-assets.js'
  * @property {Map.<string, string>} extensionURLs - map of ID => URL from project metadata. May not match extensionIDs.
  */
 
+const serializeBlock = function (block) {
+    const obj = Object.create(null);
+    obj.id = block.id;
+    obj.opcode = block.opcode;
+    obj.next = block.next;
+    obj.parent = block.parent;
+    obj.inputs = block.inputs;
+    obj.fields = block.fields;
+    obj.topLevel = block.topLevel ? block.topLevel : false;
+    obj.shadow = block.shadow;
+    if (block.topLevel) {
+        if (block.x) {
+            obj.x = Math.round(block.x);
+        }
+        if (block.y) {
+            obj.y = Math.round(block.y);
+        }
+    }
+    if (block.mutation) {
+        obj.mutation = block.mutation;
+    }
+    return obj;
+};
+
+const serializeBlocks = function (blocks) {
+    const obj = Object.create(null);
+    for (const blockID in blocks) {
+        if (!blocks.hasOwnProperty(blockID)) continue;
+        obj[blockID] = serializeBlock(blocks[blockID]);
+    }
+    return obj;
+};
+
+const serializeCostume = function (costume) {
+    const obj = Object.create(null);
+    obj.assetId = costume.assetId;
+    obj.name = costume.name;
+    obj.bitmapResolution = costume.bitmapResolution;
+    // serialize this property with the name 'md5ext' because that's
+    // what it's actually referring to. TODO runtime objects need to be
+    // updated to actually refer to this as 'md5ext' instead of 'md5'
+    // but that change should be made carefully since it is very
+    // pervasive
+    obj.md5ext = costume.md5;
+    obj.dataFormat = costume.dataFormat;
+    obj.rotationCenterX = costume.rotationCenterX;
+    obj.rotationCenterY = costume.rotationCenterY;
+    return obj;
+};
+
+const serializeSound = function (sound) {
+    const obj = Object.create(null);
+    obj.assetId = sound.assetId;
+    obj.name = sound.name;
+    obj.dataFormat = sound.dataFormat;
+    obj.format = sound.format;
+    obj.rate = sound.rate;
+    obj.sampleCount = sound.sampleCount;
+    // serialize this property with the name 'md5ext' because that's
+    // what it's actually referring to. TODO runtime objects need to be
+    // updated to actually refer to this as 'md5ext' instead of 'md5'
+    // but that change should be made carefully since it is very
+    // pervasive
+    obj.md5ext = sound.md5;
+    return obj;
+};
+
+const serializeTarget = function (target) {
+    const obj = Object.create(null);
+    obj.isStage = target.isStage;
+    obj.name = target.name;
+    obj.variables = target.variables; // This means that uids for variables will persist across saves/loads
+    obj.blocks = serializeBlocks(target.blocks);
+    obj.currentCostume = target.currentCostume;
+    obj.costumes = target.costumes.map(serializeCostume);
+    obj.sounds = target.sounds.map(serializeSound);
+    if (!obj.isStage) {
+        // Stage does not need the following properties
+        obj.visible = target.visible;
+        obj.x = target.x;
+        obj.y = target.y;
+        obj.size = target.size;
+        obj.direction = target.direction;
+        obj.draggable = target.draggable;
+        obj.rotationStyle = target.rotationStyle;
+    }
+    return obj;
+};
+
 /**
  * Serializes the specified VM runtime.
  * @param  {!Runtime} runtime VM runtime instance to be serialized.
@@ -33,7 +122,11 @@ const {deserializeCostume, deserializeSound} = require('./deserialize-assets.js'
 const serialize = function (runtime) {
     // Fetch targets
     const obj = Object.create(null);
-    obj.targets = runtime.targets.filter(target => target.isOriginal);
+    const flattenedOriginalTargets = JSON.parse(JSON.stringify(
+        runtime.targets.filter(target => target.isOriginal)));
+    obj.targets = flattenedOriginalTargets.map(t => serializeTarget(t, runtime));
+
+    // TODO Serialize monitors
 
     // Assemble metadata
     const meta = Object.create(null);
@@ -75,6 +168,7 @@ const parseScratchObject = function (object, runtime, extensions, zip) {
     }
     if (object.hasOwnProperty('blocks')) {
         for (const blockId in object.blocks) {
+            if (!object.blocks.hasOwnProperty(blockId)) continue;
             const blockJSON = object.blocks[blockId];
             blocks.createBlock(blockJSON);
 
@@ -90,6 +184,7 @@ const parseScratchObject = function (object, runtime, extensions, zip) {
     const costumePromises = (object.costumes || []).map(costumeSource => {
         // @todo: Make sure all the relevant metadata is being pulled out.
         const costume = {
+            assetId: costumeSource.assetId,
             skinId: null,
             name: costumeSource.name,
             bitmapResolution: costumeSource.bitmapResolution,
@@ -100,26 +195,41 @@ const parseScratchObject = function (object, runtime, extensions, zip) {
             costumeSource.dataFormat ||
             (costumeSource.assetType && costumeSource.assetType.runtimeFormat) || // older format
             'png'; // if all else fails, guess that it might be a PNG
-        const costumeMd5 = `${costumeSource.assetId}.${dataFormat}`;
-        costume.md5 = costumeMd5;
-        return deserializeCostume(costumeSource, runtime, zip)
-            .then(() => loadCostume(costumeMd5, costume, runtime));
+        const costumeMd5Ext = costumeSource.hasOwnProperty('md5ext') ?
+            costumeSource.md5ext : `${costumeSource.assetId}.${dataFormat}`;
+        costume.md5 = costumeMd5Ext;
+        costume.dataFormat = dataFormat;
+        // deserializeCostume should be called on the costume object we're
+        // creating above instead of the source costume object, because this way
+        // we're always loading the 'sb3' representation of the costume
+        // any translation that needs to happen will happen in the process
+        // of building up the costume object into an sb3 format
+        return deserializeCostume(costume, runtime, zip)
+            .then(() => loadCostume(costumeMd5Ext, costume, runtime));
         // Only attempt to load the costume after the deserialization
         // process has been completed
     });
     // Sounds from JSON
     const soundPromises = (object.sounds || []).map(soundSource => {
         const sound = {
+            assetId: soundSource.assetId,
             format: soundSource.format,
-            // fileUrl: soundSource.fileUrl,
             rate: soundSource.rate,
             sampleCount: soundSource.sampleCount,
-            soundID: soundSource.soundID,
             name: soundSource.name,
-            md5: soundSource.md5,
+            // TODO we eventually want this property to be called md5ext,
+            // but there are many things relying on this particular name at the
+            // moment, so this translation is very important
+            md5: soundSource.md5ext,
+            dataFormat: soundSource.dataFormat,
             data: null
         };
-        return deserializeSound(soundSource, runtime, zip)
+        // deserializeSound should be called on the sound object we're
+        // creating above instead of the source sound object, because this way
+        // we're always loading the 'sb3' representation of the costume
+        // any translation that needs to happen will happen in the process
+        // of building up the costume object into an sb3 format
+        return deserializeSound(sound, runtime, zip)
             .then(() => loadSound(sound, runtime));
         // Only attempt to load the sound after the deserialization
         // process has been completed.

src/serialization/serialize-assets.js

@@ -19,8 +19,7 @@ const serializeAssets = function (runtime, assetType) {
             const storage = runtime.storage;
             const storedAsset = storage.get(assetId);
             assetDescs.push({
-                fileName: currAsset.md5 ?
-                    currAsset.md5 : `${assetId}.${storedAsset.dataFormat}`,
+                fileName: `${assetId}.${storedAsset.dataFormat}`,
                 fileContent: storedAsset.data});
         }
     }

src/virtual-machine.js

@@ -177,34 +177,41 @@ class VirtualMachine extends EventEmitter {
     }
 
     /**
-     * Load a project from a Scratch 2.0 JSON representation.
-     * @param {?string} json JSON string representing the project.
+     * Load a Scratch project from a .sb, .sb2, .sb3 or json string.
+     * @param {string | object} input A json string, object, or ArrayBuffer representing the project to load.
      * @return {!Promise} Promise that resolves after targets are installed.
      */
-    loadProject (json) {
-        // @todo: Handle other formats, e.g., Scratch 1.4, Scratch 3.0.
-        return this.fromJSON(json);
-    }
+    loadProject (input) {
+        if (typeof input === 'object' && !(input instanceof ArrayBuffer) &&
+          !ArrayBuffer.isView(input)) {
+            // If the input is an object and not any ArrayBuffer
+            // or an ArrayBuffer view (this includes all typed arrays and DataViews)
+            // turn the object into a JSON string, because we suspect
+            // this is a project.json as an object
+            // validate expects a string or buffer as input
+            // TODO not sure if we need to check that it also isn't a data view
+            input = JSON.stringify(input);
+        }
 
-    /**
-     * Load a project from a Scratch 3.0 sb3 file containing a project json
-     * and all of the sound and costume files.
-     * @param {Buffer} inputBuffer A buffer representing the project to load.
-     * @return {!Promise} Promise that resolves after targets are installed.
-     */
-    loadProjectLocal (inputBuffer) {
-        // TODO need to handle sb2 files as well, and will possibly merge w/
-        // above function
-        return JSZip.loadAsync(inputBuffer)
-            .then(sb3File => {
-                sb3File.file('project.json').async('string')
-                    .then(json => {
-                        // TODO error handling for unpacking zip/not finding project.json
-                        json = JSON.parse(json); // TODO catch errors here (validation)
-                        return sb3.deserialize(json, this.runtime, sb3File)
-                            .then(({targets, extensions}) =>
-                                this.installTargets(targets, extensions, true));
-                    });
+        const validationPromise = new Promise((resolve, reject) => {
+            validate(input, (error, res) => {
+                if (error) {
+                    reject(error);
+                }
+                if (res) {
+                    resolve(res);
+                }
+            });
+        });
+
+        return validationPromise
+            .then(validatedInput => this.deserializeProject(validatedInput[0], validatedInput[1]))
+            .catch(error => {
+                // Intentionally rejecting here (want errors to be handled by caller)
+                if (error.hasOwnProperty('validationError')) {
+                    return Promise.reject(JSON.stringify(error));
+                }
+                return Promise.reject(error);
             });
     }
 
@@ -233,6 +240,8 @@ class VirtualMachine extends EventEmitter {
         const costumeDescs = serializeCostumes(this.runtime);
         const projectJson = this.toJSON();
 
+        // TODO want to eventually move zip creation out of here, and perhaps
+        // into scratch-storage
         const zip = new JSZip();
 
         // Put everything in a zip file
@@ -258,47 +267,40 @@ class VirtualMachine extends EventEmitter {
         return JSON.stringify(sb3.serialize(this.runtime));
     }
 
+    // TODO do we still need this function? Keeping it here so as not to introduce
+    // a breaking change.
     /**
      * Load a project from a Scratch JSON representation.
      * @param {string} json JSON string representing a project.
      * @returns {Promise} Promise that resolves after the project has loaded
      */
     fromJSON (json) {
+        log.warning('fromJSON is now just a wrapper around loadProject, please use that function instead.');
+        return this.loadProject(json);
+    }
+
+    /**
+     * Load a project from a Scratch JSON representation.
+     * @param {string} projectJSON JSON string representing a project.
+     * @param {?JSZip} zip Optional zipped project containing assets to be loaded.
+     * @returns {Promise} Promise that resolves after the project has loaded
+     */
+    deserializeProject (projectJSON, zip) {
         // Clear the current runtime
         this.clear();
 
-        // Validate & parse
-        if (typeof json !== 'string' && typeof json !== 'object') {
-            throw new Error('Failed to parse project. Invalid type supplied to fromJSON.');
-        }
-
-        // Establish version, deserialize, and load into runtime
-        // @todo Support Scratch 1.4
-        // @todo This is an extremely naïve / dangerous way of determining version.
-        //       See `scratch-parser` for a more sophisticated validation
-        //       methodology that should be adapted for use here
-        let deserializer;
-        let validatedProject;
-        const possibleSb3 = typeof json === 'string' ? JSON.parse(json) : json;
-        if ((typeof possibleSb3.meta !== 'undefined') && (typeof possibleSb3.meta.semver !== 'undefined')) {
-            deserializer = sb3;
-            validatedProject = possibleSb3;
-        } else {
-            // scratch-parser expects a json string or a buffer
-            const possibleSb2 = typeof json === 'object' ? JSON.stringify(json) : json;
-            validate(possibleSb2, (err, project) => {
-                if (err) {
-                    throw new Error(
-                        `The given project could not be validated, parsing failed with error: ${JSON.stringify(err)}`);
-
-                } else {
-                    deserializer = sb2;
-                    validatedProject = project;
-                }
-            });
-        }
-
-        return deserializer.deserialize(validatedProject, this.runtime)
+        const runtime = this.runtime;
+        const deserializePromise = function () {
+            const projectVersion = projectJSON.projectVersion;
+            if (projectVersion === 2) {
+                return sb2.deserialize(projectJSON, runtime);
+            }
+            if (projectVersion === 3) {
+                return sb3.deserialize(projectJSON, runtime, zip);
+            }
+            return Promise.reject('Unable to verify Scratch Project version.');
+        };
+        return deserializePromise()
             .then(({targets, extensions}) =>
                 this.installTargets(targets, extensions, true));
     }

test/unit/serialization_sb3.js

@@ -5,11 +5,13 @@ const demoSb3 = require('../fixtures/demo.json');
 
 test('serialize', t => {
     const vm = new VirtualMachine();
-    vm.fromJSON(JSON.stringify(demoSb3));
-    const result = sb3.serialize(vm.runtime);
-    // @todo Analyze
-    t.type(JSON.stringify(result), 'string');
-    t.end();
+    vm.loadProject(JSON.stringify(demoSb3))
+        .then(() => {
+            const result = sb3.serialize(vm.runtime);
+            // @todo Analyze
+            t.type(JSON.stringify(result), 'string');
+            t.end();
+        });
 });
 
 test('deserialize', t => {