Merge pull request #1737 from fsih/textLayer

Support text layer in sb2 files

src/import/load-costume.js

@@ -1,8 +1,8 @@
 const StringUtil = require('../util/string-util');
 const log = require('../util/log');
 
-const loadVector_ = function (costume, costumeAsset, runtime, rotationCenter, optVersion) {
+const loadVector_ = function (costume, runtime, rotationCenter, optVersion) {
-    let svgString = costumeAsset.decodeText();
+    let svgString = costume.asset.decodeText();
     // SVG Renderer load fixes "quirks" associated with Scratch 2 projects
     if (optVersion && optVersion === 2 && !runtime.v2SvgAdapter) {
         log.error('No V2 SVG adapter present; SVGs may not render correctly.');
@@ -30,34 +30,120 @@ const loadVector_ = function (costume, costumeAsset, runtime, rotationCenter, op
     return Promise.resolve(costume);
 };
 
-const loadBitmap_ = function (costume, costumeAsset, runtime, rotationCenter) {
+/**
+ * Return a promise to fetch a bitmap from storage and return it as a canvas
+ * If the costume has bitmapResolution 1, it will be converted to bitmapResolution 2 here (the standard for Scratch 3)
+ * If the costume has a text layer asset, which is a text part from Scratch 1.4, then this function
+ * will merge the two image assets. See the issue LLK/scratch-vm#672 for more information.
+ * @param {!object} costume - the Scratch costume object.
+ * @param {!Runtime} runtime - Scratch runtime, used to access the v2BitmapAdapter
+ * @param {?object} rotationCenter - optionally passed in coordinates for the center of rotation for the image. If
+ *     none is given, the rotation center of the costume will be set to the middle of the costume later on.
+ * @property {number} costume.bitmapResolution - the resolution scale for a bitmap costume.
+ * @returns {?Promise} - a promise which will resolve to an object {canvas, rotationCenter, assetMatchesBase},
+ *     or reject on error.
+ *     assetMatchesBase is true if the asset matches the base layer; false if it required adjustment
+ */
+const fetchBitmapCanvas_ = function (costume, runtime, rotationCenter) {
+    if (!costume || !costume.asset) {
+        return Promise.reject('Costume load failed. Assets were missing.');
+    }
+    if (!runtime.v2BitmapAdapter) {
+        return Promise.reject('No V2 Bitmap adapter present.');
+    }
+
     return new Promise((resolve, reject) => {
-        const imageElement = new Image();
+        const baseImageElement = new Image();
+        let textImageElement;
+
+        // We need to wait for 2 images total to load. loadedOne will be true when one
+        // is done, and we are just waiting for one more.
+        let loadedOne = false;
+
         const onError = function () {
             // eslint-disable-next-line no-use-before-define
             removeEventListeners();
-            reject('Image load failed');
+            reject('Costume load failed. Asset could not be read.');
         };
         const onLoad = function () {
+            if (loadedOne) {
                 // eslint-disable-next-line no-use-before-define
                 removeEventListeners();
-            resolve(imageElement);
+                resolve([baseImageElement, textImageElement]);
+            } else {
+                loadedOne = true;
+            }
         };
+
         const removeEventListeners = function () {
-            imageElement.removeEventListener('error', onError);
+            baseImageElement.removeEventListener('error', onError);
-            imageElement.removeEventListener('load', onLoad);
+            baseImageElement.removeEventListener('load', onLoad);
+            if (textImageElement) {
+                textImageElement.removeEventListener('error', onError);
+                textImageElement.removeEventListener('load', onLoad);
+            }
         };
-        imageElement.addEventListener('error', onError);
+
-        imageElement.addEventListener('load', onLoad);
+        baseImageElement.addEventListener('load', onLoad);
-        const src = costumeAsset.encodeDataURI();
+        baseImageElement.addEventListener('error', onError);
-        if (costume.bitmapResolution === 1 && !runtime.v2BitmapAdapter) {
+        if (costume.textLayerAsset) {
-            log.error('No V2 bitmap adapter present; bitmaps may not render correctly.');
+            textImageElement = new Image();
-        } else if (costume.bitmapResolution === 1) {
+            textImageElement.addEventListener('load', onLoad);
-            runtime.v2BitmapAdapter.convertResolution1Bitmap(src, (error, dataURI) => {
+            textImageElement.addEventListener('error', onError);
-                if (error) {
+            textImageElement.src = costume.textLayerAsset.encodeDataURI();
-                    log.error(error);
+        } else {
-                } else if (dataURI) {
+            loadedOne = true;
-                    // Put back into storage
+        }
+        baseImageElement.src = costume.asset.encodeDataURI();
+    }).then(imageElements => {
+        const [baseImageElement, textImageElement] = imageElements;
+
+        let canvas = document.createElement('canvas');
+        const scale = costume.bitmapResolution === 1 ? 2 : 1;
+        canvas.width = baseImageElement.width;
+        canvas.height = baseImageElement.height;
+
+        const ctx = canvas.getContext('2d');
+        ctx.drawImage(baseImageElement, 0, 0);
+        if (textImageElement) {
+            ctx.drawImage(textImageElement, 0, 0);
+        }
+        if (scale !== 1) {
+            canvas = runtime.v2BitmapAdapter.resize(canvas, canvas.width * scale, canvas.height * scale);
+        }
+
+        // By scaling, we've converted it to bitmap resolution 2
+        if (rotationCenter) {
+            rotationCenter[0] = rotationCenter[0] * scale;
+            rotationCenter[1] = rotationCenter[1] * scale;
+            costume.rotationCenterX = rotationCenter[0];
+            costume.rotationCenterY = rotationCenter[1];
+        }
+        costume.bitmapResolution = 2;
+
+        return {
+            canvas: canvas,
+            rotationCenter: rotationCenter,
+            // True if the asset matches the base layer; false if it required adjustment
+            assetMatchesBase: scale === 1 && !textImageElement
+        };
+    })
+        .finally(() => {
+            // Clean up the costume object
+            delete costume.textLayerMD5;
+            delete costume.textLayerAsset;
+        });
+};
+
+const loadBitmap_ = function (costume, runtime, rotationCenter) {
+    return fetchBitmapCanvas_(costume, runtime, rotationCenter).then(fetched => new Promise(resolve => {
+        rotationCenter = fetched.rotationCenter;
+
+        const updateCostumeAsset = function (dataURI) {
+            if (!runtime.v2BitmapAdapter) {
+                return Promise.reject('No V2 Bitmap adapter present.');
+            }
+
             const storage = runtime.storage;
             costume.asset = storage.createAsset(
                 storage.AssetType.ImageBitmap,
@@ -68,26 +154,16 @@ const loadBitmap_ = function (costume, costumeAsset, runtime, rotationCenter) {
             );
             costume.assetId = costume.asset.assetId;
             costume.md5 = `${costume.assetId}.${costume.dataFormat}`;
+        };
+
+        if (!fetched.assetMatchesBase) {
+            updateCostumeAsset(fetched.canvas.toDataURL());
         }
-                // Regardless of if conversion succeeds, convert it to bitmap resolution 2,
+        resolve(fetched.canvas);
-                // since all code from here on will assume that.
+    }))
-                if (rotationCenter) {
+        .then(canvas => {
-                    rotationCenter[0] = rotationCenter[0] * 2;
-                    rotationCenter[1] = rotationCenter[1] * 2;
-                    costume.rotationCenterX = rotationCenter[0];
-                    costume.rotationCenterY = rotationCenter[1];
-                }
-                costume.bitmapResolution = 2;
-                // Use original src if conversion fails.
-                // The image will appear half-sized.
-                imageElement.src = dataURI ? dataURI : src;
-            });
-        } else {
-            imageElement.src = src;
-        }
-    }).then(imageElement => {
             // createBitmapSkin does the right thing if costume.bitmapResolution or rotationCenter are undefined...
-        costume.skinId = runtime.renderer.createBitmapSkin(imageElement, costume.bitmapResolution, rotationCenter);
+            costume.skinId = runtime.renderer.createBitmapSkin(canvas, costume.bitmapResolution, rotationCenter);
             const renderSize = runtime.renderer.getSkinSize(costume.skinId);
             costume.size = [renderSize[0] * 2, renderSize[1] * 2]; // Actual size, since all bitmaps are resolution 2
 
@@ -110,14 +186,14 @@ const loadBitmap_ = function (costume, costumeAsset, runtime, rotationCenter) {
  * @property {number} rotationCenterX - the X component of the costume's origin.
  * @property {number} rotationCenterY - the Y component of the costume's origin.
  * @property {number} [bitmapResolution] - the resolution scale for a bitmap costume.
- * @param {!Asset} costumeAsset - the asset of the costume loaded from storage.
+ * @property {!Asset} costume.asset - the asset of the costume loaded from storage.
  * @param {!Runtime} runtime - Scratch runtime, used to access the storage module.
  * @param {?int} optVersion - Version of Scratch that the costume comes from. If this is set
  *     to 2, scratch 3 will perform an upgrade step to handle quirks in SVGs from Scratch 2.0.
  * @returns {?Promise} - a promise which will resolve after skinId is set, or null on error.
  */
-const loadCostumeFromAsset = function (costume, costumeAsset, runtime, optVersion) {
+const loadCostumeFromAsset = function (costume, runtime, optVersion) {
-    costume.assetId = costumeAsset.assetId;
+    costume.assetId = costume.asset.assetId;
     const renderer = runtime.renderer;
     if (!renderer) {
         log.error('No rendering module present; cannot load costume: ', costume.name);
@@ -131,16 +207,16 @@ const loadCostumeFromAsset = function (costume, costumeAsset, runtime, optVersio
             typeof costume.rotationCenterY === 'number' && !isNaN(costume.rotationCenterY)) {
         rotationCenter = [costume.rotationCenterX, costume.rotationCenterY];
     }
-    if (costumeAsset.assetType === AssetType.ImageVector) {
+    if (costume.asset.assetType === AssetType.ImageVector) {
-        return loadVector_(costume, costumeAsset, runtime, rotationCenter, optVersion);
+        return loadVector_(costume, runtime, rotationCenter, optVersion);
     }
-    return loadBitmap_(costume, costumeAsset, runtime, rotationCenter, optVersion);
+    return loadBitmap_(costume, runtime, rotationCenter, optVersion);
 };
 
 /**
  * Load a costume's asset into memory asynchronously.
  * Do not call this unless there is a renderer attached.
- * @param {string} md5ext - the MD5 and extension of the costume to be loaded.
+ * @param {!string} md5ext - the MD5 and extension of the costume to be loaded.
  * @param {!object} costume - the Scratch costume object.
  * @property {int} skinId - the ID of the costume's render skin, once installed.
  * @property {number} rotationCenterX - the X component of the costume's origin.
@@ -152,23 +228,44 @@ const loadCostumeFromAsset = function (costume, costumeAsset, runtime, optVersio
  * @returns {?Promise} - a promise which will resolve after skinId is set, or null on error.
  */
 const loadCostume = function (md5ext, costume, runtime, optVersion) {
+    const idParts = StringUtil.splitFirst(md5ext, '.');
+    const md5 = idParts[0];
+    const ext = idParts[1].toLowerCase();
+    costume.dataFormat = ext;
+
+    if (costume.asset) {
+        // Costume comes with asset. It could be coming from camera, image upload, drag and drop, or file
+        return loadCostumeFromAsset(costume, runtime, optVersion);
+    }
+
+    // Need to load the costume from storage. The server should have a reference to this md5.
     if (!runtime.storage) {
         log.error('No storage module present; cannot load costume asset: ', md5ext);
         return Promise.resolve(costume);
     }
 
     const AssetType = runtime.storage.AssetType;
-    const idParts = StringUtil.splitFirst(md5ext, '.');
-    const md5 = idParts[0];
-    const ext = idParts[1].toLowerCase();
     const assetType = (ext === 'svg') ? AssetType.ImageVector : AssetType.ImageBitmap;
-    costume.dataFormat = ext;
+
-    return (
+    const costumePromise = runtime.storage.load(assetType, md5, ext);
-        (costume.asset && Promise.resolve(costume.asset)) ||
+    if (!costumePromise) {
-        runtime.storage.load(assetType, md5, ext)
+        log.error(`Couldn't fetch costume asset: ${md5ext}`);
-    ).then(costumeAsset => {
+        return;
-        costume.asset = costumeAsset;
+    }
-        return loadCostumeFromAsset(costume, costumeAsset, runtime, optVersion);
+
+    let textLayerPromise;
+    if (costume.textLayerMD5) {
+        textLayerPromise = runtime.storage.load(AssetType.ImageBitmap, costume.textLayerMD5, 'png');
+    } else {
+        textLayerPromise = Promise.resolve(null);
+    }
+
+    return Promise.all([costumePromise, textLayerPromise]).then(assetArray => {
+        costume.asset = assetArray[0];
+        if (assetArray[1]) {
+            costume.textLayerAsset = assetArray[1];
+        }
+        return loadCostumeFromAsset(costume, runtime, optVersion);
     })
         .catch(e => {
             log.error(e);

src/serialization/deserialize-assets.js

@@ -56,11 +56,13 @@ const deserializeSound = function (sound, runtime, zip, assetFileName) {
  * @param {string} assetFileName Optional file name for the given asset
  * (sb2 files have filenames of the form [int].[ext],
  * sb3 files have filenames of the form [md5].[ext])
+ * @param {string} textLayerFileName Optional file name for the given asset's text layer
+ * (sb2 only; files have filenames of the form [int].png)
  * @return {Promise} Promise that resolves after the described costume has been stored
  * into the runtime storage cache, the costume was already stored, or an error has
  * occurred.
  */
-const deserializeCostume = function (costume, runtime, zip, assetFileName) {
+const deserializeCostume = function (costume, runtime, zip, assetFileName, textLayerFileName) {
     const storage = runtime.storage;
     const assetId = costume.assetId;
     const fileName = assetFileName ? assetFileName :
@@ -79,7 +81,9 @@ const deserializeCostume = function (costume, runtime, zip, assetFileName) {
             costume.asset.dataFormat,
             new Uint8Array(Object.keys(costume.asset.data).map(key => costume.asset.data[key])),
             costume.asset.assetId
-        ));
+        )).then(asset => {
+            costume.asset = asset;
+        });
     }
 
     if (!zip) {
@@ -106,13 +110,42 @@ const deserializeCostume = function (costume, runtime, zip, assetFileName) {
         return Promise.resolve(null);
     }
 
-    return costumeFile.async('uint8array').then(data => storage.createAsset(
+    // textLayerMD5 exists if there is a text layer, which is a png of text from Scratch 1.4
+    // that was opened in Scratch 2.0. In this case, set costume.textLayerAsset.
+    let textLayerFilePromise;
+    if (costume.textLayerMD5) {
+        const textLayerFile = zip.file(textLayerFileName);
+        if (!textLayerFile) {
+            log.error(`Could not find text layer file associated with the ${costume.name} costume.`);
+            return Promise.resolve(null);
+        }
+        textLayerFilePromise = textLayerFile.async('uint8array')
+            .then(data => storage.createAsset(
+                storage.AssetType.ImageBitmap,
+                'png',
+                data,
+                costume.textLayerMD5
+            ))
+            .then(asset => {
+                costume.textLayerAsset = asset;
+            });
+    } else {
+        textLayerFilePromise = Promise.resolve(null);
+    }
+
+    return Promise.all([textLayerFilePromise,
+        costumeFile.async('uint8array')
+            .then(data => storage.createAsset(
                 assetType,
                 // TODO eventually we want to map non-png's to their actual file types?
                 costumeFormat,
                 data,
                 assetId
-    ));
+            ))
+            .then(asset => {
+                costume.asset = asset;
+            })
+    ]);
 };
 
 module.exports = {

src/serialization/sb2.js

@@ -445,15 +445,16 @@ const parseScratchObject = function (object, runtime, extensions, topLevel, zip)
             const ext = idParts[1].toLowerCase();
             costume.dataFormat = ext;
             costume.assetId = md5;
+            if (costumeSource.textLayerMD5) {
+                costume.textLayerMD5 = StringUtil.splitFirst(costumeSource.textLayerMD5, '.')[0];
+            }
             // If there is no internet connection, or if the asset is not in storage
             // for some reason, and we are doing a local .sb2 import, (e.g. zip is provided)
             // the file name of the costume should be the baseLayerID followed by the file ext
             const assetFileName = `${costumeSource.baseLayerID}.${ext}`;
-            costumePromises.push(deserializeCostume(costume, runtime, zip, assetFileName)
+            const textLayerFileName = costumeSource.textLayerID ? `${costumeSource.textLayerID}.png` : null;
-                .then(asset => {
+            costumePromises.push(deserializeCostume(costume, runtime, zip, assetFileName, textLayerFileName)
-                    costume.asset = asset;
+                .then(() => loadCostume(costume.md5, costume, runtime, 2 /* optVersion */))
-                    return loadCostume(costume.md5, costume, runtime, 2 /* optVersion */);
-                })
             );
         }
     }

src/serialization/sb3.js

@@ -881,10 +881,7 @@ const parseScratchObject = function (object, runtime, extensions, zip) {
         // any translation that needs to happen will happen in the process
         // of building up the costume object into an sb3 format
         return deserializeCostume(costume, runtime, zip)
-            .then(asset => {
+            .then(() => loadCostume(costumeMd5Ext, costume, runtime));
-                costume.asset = asset;
-                return loadCostume(costumeMd5Ext, costume, runtime);
-            });
         // Only attempt to load the costume after the deserialization
         // process has been completed
     });