Add tests, fix problems found by those tests

Changes include: - Use `bin-loader` instead of `arraybuffer-loader` since the latter is only compatible with browsers, not with Node itself. - Use `got` instead of `xhr` since `xhr` only works in browsers and `got` is nicer anyway. - Add `json-loader` to Webpack config since `got` requires it. - Use lower-case values in `DataFormat` to match canonical file names. - Update `ScratchStorage.addWebSource` to match `WebHelper`'s new API. - Fix an error in the web helper which was causing an infinite loop on error.
2025-06-17 01:31:13 -04:00 · 2016-12-08 16:38:38 -08:00 · 2016-12-08 16:38:38 -08:00 · 2b28f27049
commit 2b28f27049
parent fe3acfa8a2
10 changed files with 202 additions and 43 deletions
--- a/package.json
+++ b/package.json
@ -23,19 +23,21 @@
    "watch": "./node_modules/.bin/webpack --progress --colors --watch-poll"
  },
  "devDependencies": {
-    "arraybuffer-loader": "^0.2.1",
    "babel-core": "^6.17.0",
    "babel-eslint": "^7.0.0",
    "babel-loader": "^6.2.5",
    "babel-polyfill": "^6.16.0",
    "babel-preset-es2015": "^6.16.0",
+    "bin-loader": "^0.1.0",
    "debug": "^2.2.0",
    "eslint": "^3.8.1",
    "eslint-config-scratch": "^3.1.0",
    "eslint-plugin-react": "^6.4.1",
    "file-loader": "^0.9.0",
+    "got": "^5.7.1",
+    "json-loader": "^0.5.4",
    "localforage": "^1.4.3",
-    "webpack": "^1.13.2",
-    "xhr": "^2.2.2"
+    "tap": "^8.0.1",
+    "webpack": "^1.13.2"
  }
 }
--- a/src/BuiltinHelper.js
+++ b/src/BuiltinHelper.js
@ -10,7 +10,7 @@ const Helper = require('./Helper');
 * @property {AssetType} type - The type of the asset.
 * @property {DataFormat} format - The format of the asset's data.
 * @property {string} id - The asset's unique ID.
- * @property {string} data - The asset's data in string form.
+ * @property {Uint8Array} data - The asset's data in string form.
 */

 /**
@ -21,19 +21,19 @@ const DefaultAssets = [
        type: AssetType.ImageBitmap,
        format: DataFormat.PNG,
        id: null,
-        data: require('arraybuffer!./builtins/defaultBitmap.png') // eslint-disable-line global-require
+        data: require('bin!./builtins/defaultBitmap.png') // eslint-disable-line global-require
    },
    {
        type: AssetType.Sound,
        format: DataFormat.WAV,
        id: null,
-        data: require('arraybuffer!./builtins/defaultSound.wav') // eslint-disable-line global-require
+        data: require('bin!./builtins/defaultSound.wav') // eslint-disable-line global-require
    },
    {
        type: AssetType.ImageVector,
        format: DataFormat.SVG,
        id: null,
-        data: require('arraybuffer!./builtins/defaultVector.svg') // eslint-disable-line global-require
+        data: require('bin!./builtins/defaultVector.svg') // eslint-disable-line global-require
    }
 ];

@ -95,12 +95,12 @@ class BuiltinHelper extends Helper {
            if (typeBucket.hasOwnProperty(assetId)) {
                /** @type{BuiltinAssetRecord} */
                const assetRecord = typeBucket[assetId];
-                const asset =
-                    new Asset(assetRecord.type, assetRecord.id, assetRecord.format, new Uint8Array(assetRecord.data));
+                const assetData = new Uint8Array(assetRecord.data.buffer);
+                const asset = new Asset(assetRecord.type, assetRecord.id, assetRecord.format, assetData);
                return Promise.resolve(asset);
            }
        }
-        return Promise.reject(new Error(`No builtin asset of type ${assetType} for ID ${assetId}`));
+        return Promise.reject(new Error(`No builtin asset of type ${assetType.name} for ID ${assetId}`));
    }
 }

--- a/src/DataFormat.js
+++ b/src/DataFormat.js
@ -3,11 +3,11 @@
 * @type {Object.<string,string>}
 */
 const DataFormat = {
-    JSON: 'JSON',
-    PNG: 'PNG',
-    SB2: 'SB2',
-    SVG: 'SVG',
-    WAV: 'WAV'
+    JSON: 'json',
+    PNG: 'png',
+    SB2: 'sb2',
+    SVG: 'svg',
+    WAV: 'wav'
 };

 module.exports = DataFormat;
--- a/src/ScratchStorage.js
+++ b/src/ScratchStorage.js
@ -1,4 +1,3 @@
-const Asset = require('./Asset');
 const BuiltinHelper = require('./BuiltinHelper');
 const LocalHelper = require('./LocalHelper');
 const WebHelper = require('./WebHelper');
@ -17,13 +16,10 @@ class ScratchStorage {
    /**
     * Register a web-based source for assets. Sources will be checked in order of registration.
     * @param {Array.<AssetType>} types - The types of asset provided by this source.
-     * @param {string} urlPattern - The URL pattern to use when loading assets from this source.
-     * The following sub-strings, if found in the URL pattern, will be replaced with properties of the requested asset:
-     * - '{ID}' will be replaced with the ID of the requested asset.
-     * - '{EXT}' will be replaced with the file extension of the requested asset (such as 'svg').
+     * @param {UrlFunction} urlFunction - A function which computes a URL from an Asset.
     */
-    addWebSource (types, urlPattern) {
-        this.webHelper.addSource(types, urlPattern);
+    addWebSource (types, urlFunction) {
+        this.webHelper.addSource(types, urlFunction);
    }

    /**
@ -86,10 +82,4 @@ class ScratchStorage {
    }
 }

-/**
- * Export the Asset class so that clients can use it for asset-save requests.
- * @type {Asset}
- */
-ScratchStorage.Asset = Asset;
-
 module.exports = ScratchStorage;
--- a/src/WebHelper.js
+++ b/src/WebHelper.js
@ -1,10 +1,12 @@
+const got = require('got');
+
 const Asset = require('./Asset');
 const Helper = require('./Helper');
-const xhr = require('xhr');

 /**
 * @typedef {function} UrlFunction - A function which computes a URL from asset information.
 * @param {Asset} - The asset for which the URL should be computed.
+ * @returns {string} - The URL for the asset.
 */

 class WebHelper extends Helper {
@ -53,8 +55,9 @@ class WebHelper extends Helper {
                /** @type {UrlFunction} */
                let urlFunction;

-                for (; sourceIndex < sources.length; ++sourceIndex) {
+                while (sourceIndex < sources.length) {
                    const source = sources[sourceIndex];
+                    ++sourceIndex;
                    if (source.types.indexOf(assetType) >= 0) {
                        urlFunction = source.urlFunction;
                        break;
@ -62,21 +65,28 @@ class WebHelper extends Helper {
                }

                if (urlFunction) {
+                    const options = {
+                        encoding: null // return body as Buffer
+                    };
                    const url = urlFunction(asset);
-                    xhr({
-                        uri: url
-                    }, (error, response, body) => {
-                        if (error) {
+                    got(url, options).then(
+                        response => {
+                            if (response.status < 200 || response.status >= 300) {
+                                errors.push({url: url, result: response});
+                                tryNextSource();
+                            } else {
+                                /** @type {Buffer} */
+                                const buffer = response.body;
+
+                                // Convert from Buffer to Uint8Array, assuming Node 4.x+ or compatible Webpack
+                                asset.data = new Uint8Array(buffer.buffer);
+                                fulfill(asset);
+                            }
+                        },
+                        error => {
                            errors.push({url: url, result: error});
                            tryNextSource();
-                        } else if (response.status < 200 || response.status >= 300) {
-                            errors.push({url: url, result: response});
-                            tryNextSource();
-                        } else {
-                            asset.data = body;
-                            fulfill(asset);
-                        }
-                    });
+                        });
                } else if (errors.length > 0) {
                    reject(errors);
                } else {
--- a/src/index.js
+++ b/src/index.js
@ -1,7 +1,12 @@
+const Asset = require('./Asset');
+const AssetType = require('./AssetType');
 const ScratchStorage = require('./ScratchStorage');

 /**
 * Export for use with NPM & Node.js.
 * @type {ScratchStorage}
 */
-module.exports = ScratchStorage;
+module.exports = Object.assign(ScratchStorage, {
+    Asset: Asset,
+    AssetType: AssetType
+});
--- a/test/.eslintrc.js
+++ b/test/.eslintrc.js
@ -0,0 +1,3 @@
+module.exports = {
+    extends: ['scratch/node', 'scratch/es6']
+};
--- a/test/integration/download-known-assets.js
+++ b/test/integration/download-known-assets.js
@ -0,0 +1,94 @@
+const crypto = require('crypto');
+const test = require('tap').test;
+
+const ScratchStorage = require('../../dist/node/scratch-storage');
+const {Asset, AssetType} = ScratchStorage;
+
+/**
+ *
+ * @type {AssetTestInfo[]}
+ * @typedef {object} AssetTestInfo
+ * @property {AssetType} type - The type of the asset.
+ * @property {string} id - The asset's unique ID.
+ */
+const testAssets = [
+    {
+        type: AssetType.Project,
+        id: '117504922',
+        md5: null // don't check MD5 for project without revision ID
+    },
+    {
+        type: AssetType.Project,
+        id: '117504922.d6ae1ffb76f2bc83421cd3f40fc4fd57',
+        md5: '1225460702e149727de28bff4cfd9e23'
+    },
+    {
+        type: AssetType.ImageVector,
+        id: 'f88bf1935daea28f8ca098462a31dbb0', // cat1-a
+        md5: 'f88bf1935daea28f8ca098462a31dbb0'
+    },
+    {
+        type: AssetType.ImageBitmap,
+        id: '7e24c99c1b853e52f8e7f9004416fa34', // squirrel
+        md5: '7e24c99c1b853e52f8e7f9004416fa34'
+    },
+    {
+        type: AssetType.Sound,
+        id: '83c36d806dc92327b9e7049a565c6bff', // meow
+        md5: '83c36d806dc92327b9e7049a565c6bff' // wat
+    }
+];
+
+let storage;
+test('constructor', t => {
+    storage = new ScratchStorage();
+    t.type(storage, ScratchStorage);
+    t.end();
+});
+
+test('addWebSource', t => {
+    t.doesNotThrow(() => {
+        storage.addWebSource(
+            [AssetType.Project],
+            asset => {
+                const [projectId, revision] = asset.assetId.split('.');
+                return revision ?
+                    `https://cdn.projects.scratch.mit.edu/internalapi/project/${projectId}/get/${revision}` :
+                    `https://cdn.projects.scratch.mit.edu/internalapi/project/${projectId}/get/`;
+            });
+    });
+    t.doesNotThrow(() => {
+        storage.addWebSource(
+            [AssetType.ImageVector, AssetType.ImageBitmap, AssetType.Sound],
+            asset => `https://cdn.assets.scratch.mit.edu/internalapi/asset/${asset.assetId}.${asset.assetType.runtimeFormat}/get/`
+        );
+    });
+    t.end();
+});
+
+test('load', t => {
+    const promises = [];
+    for (let i = 0; i < testAssets.length; ++i) {
+        const assetInfo = testAssets[i];
+
+        const promise = storage.load(assetInfo.type, assetInfo.id);
+        t.type(promise, 'Promise');
+
+        promises.push(promise);
+
+        promise.then(asset => {
+            t.type(asset, Asset);
+            t.strictEqual(asset.assetId, assetInfo.id);
+            t.strictEqual(asset.assetType, assetInfo.type);
+            t.ok(asset.data.length);
+
+            if (assetInfo.md5) {
+                const hash = crypto.createHash('md5');
+                hash.update(asset.data);
+                t.strictEqual(hash.digest('hex'), assetInfo.md5);
+            }
+        });
+    }
+
+    return Promise.all(promises);
+});
--- a/test/unit/load-default-assets.js
+++ b/test/unit/load-default-assets.js
@ -0,0 +1,51 @@
+const crypto = require('crypto');
+const test = require('tap').test;
+
+const ScratchStorage = require('../../dist/node/scratch-storage');
+const {Asset, AssetType} = ScratchStorage;
+
+const defaultAssetTypes = [AssetType.ImageBitmap, AssetType.ImageVector, AssetType.Sound];
+const defaultIds = {};
+
+let storage;
+test('constructor', t => {
+    storage = new ScratchStorage();
+    t.type(storage, ScratchStorage);
+    t.end();
+});
+
+test('getDefaultAssetId', t => {
+    for (let i = 0; i < defaultAssetTypes.length; ++i) {
+        const assetType = defaultAssetTypes[i];
+        const id = storage.getDefaultAssetId(assetType);
+        t.type(id, 'string');
+        defaultIds[assetType.name] = id;
+    }
+    t.end();
+});
+
+test('load', t => {
+    const promises = [];
+    for (let i = 0; i < defaultAssetTypes.length; ++i) {
+        const assetType = defaultAssetTypes[i];
+        const id = defaultIds[assetType.name];
+
+        const promise = storage.load(assetType, id);
+        t.type(promise, 'Promise');
+
+        promises.push(promise);
+
+        promise.then(asset => {
+            t.type(asset, Asset);
+            t.strictEqual(asset.assetId, id);
+            t.strictEqual(asset.assetType, assetType);
+            t.ok(asset.data.length);
+
+            const hash = crypto.createHash('md5');
+            hash.update(asset.data);
+            t.strictEqual(hash.digest('hex'), id);
+        });
+    }
+
+    return Promise.all(promises);
+});
--- a/webpack.config.js
+++ b/webpack.config.js
@ -13,6 +13,10 @@ const base = {
                query: {
                    presets: ['es2015']
                }
+            },
+            {
+                test: /\.json$/,
+                loader: 'json-loader'
            }
        ]
    },