Add tests, fix problems found by those tests

Changes include:
- Use `bin-loader` instead of `arraybuffer-loader` since the latter is
  only compatible with browsers, not with Node itself.
- Use `got` instead of `xhr` since `xhr` only works in browsers and
  `got` is nicer anyway.
- Add `json-loader` to Webpack config since `got` requires it.
- Use lower-case values in `DataFormat` to match canonical file names.
- Update `ScratchStorage.addWebSource` to match `WebHelper`'s new API.
- Fix an error in the web helper which was causing an infinite loop on
  error.

package.json

@@ -23,19 +23,21 @@
     "watch": "./node_modules/.bin/webpack --progress --colors --watch-poll"
   },
   "devDependencies": {
-    "arraybuffer-loader": "^0.2.1",
     "babel-core": "^6.17.0",
     "babel-eslint": "^7.0.0",
     "babel-loader": "^6.2.5",
     "babel-polyfill": "^6.16.0",
     "babel-preset-es2015": "^6.16.0",
+    "bin-loader": "^0.1.0",
     "debug": "^2.2.0",
     "eslint": "^3.8.1",
     "eslint-config-scratch": "^3.1.0",
     "eslint-plugin-react": "^6.4.1",
     "file-loader": "^0.9.0",
+    "got": "^5.7.1",
+    "json-loader": "^0.5.4",
     "localforage": "^1.4.3",
-    "webpack": "^1.13.2",
+    "tap": "^8.0.1",
-    "xhr": "^2.2.2"
+    "webpack": "^1.13.2"
   }
 }

src/BuiltinHelper.js

@@ -10,7 +10,7 @@ const Helper = require('./Helper');
  * @property {AssetType} type - The type of the asset.
  * @property {DataFormat} format - The format of the asset's data.
  * @property {string} id - The asset's unique ID.
- * @property {string} data - The asset's data in string form.
+ * @property {Uint8Array} data - The asset's data in string form.
  */
 
 /**
@@ -21,19 +21,19 @@ const DefaultAssets = [
         type: AssetType.ImageBitmap,
         format: DataFormat.PNG,
         id: null,
-        data: require('arraybuffer!./builtins/defaultBitmap.png') // eslint-disable-line global-require
+        data: require('bin!./builtins/defaultBitmap.png') // eslint-disable-line global-require
     },
     {
         type: AssetType.Sound,
         format: DataFormat.WAV,
         id: null,
-        data: require('arraybuffer!./builtins/defaultSound.wav') // eslint-disable-line global-require
+        data: require('bin!./builtins/defaultSound.wav') // eslint-disable-line global-require
     },
     {
         type: AssetType.ImageVector,
         format: DataFormat.SVG,
         id: null,
-        data: require('arraybuffer!./builtins/defaultVector.svg') // eslint-disable-line global-require
+        data: require('bin!./builtins/defaultVector.svg') // eslint-disable-line global-require
     }
 ];
 
@@ -95,12 +95,12 @@ class BuiltinHelper extends Helper {
             if (typeBucket.hasOwnProperty(assetId)) {
                 /** @type{BuiltinAssetRecord} */
                 const assetRecord = typeBucket[assetId];
-                const asset =
+                const assetData = new Uint8Array(assetRecord.data.buffer);
-                    new Asset(assetRecord.type, assetRecord.id, assetRecord.format, new Uint8Array(assetRecord.data));
+                const asset = new Asset(assetRecord.type, assetRecord.id, assetRecord.format, assetData);
                 return Promise.resolve(asset);
             }
         }
-        return Promise.reject(new Error(`No builtin asset of type ${assetType} for ID ${assetId}`));
+        return Promise.reject(new Error(`No builtin asset of type ${assetType.name} for ID ${assetId}`));
     }
 }
 

src/DataFormat.js

@@ -3,11 +3,11 @@
  * @type {Object.<string,string>}
  */
 const DataFormat = {
-    JSON: 'JSON',
+    JSON: 'json',
-    PNG: 'PNG',
+    PNG: 'png',
-    SB2: 'SB2',
+    SB2: 'sb2',
-    SVG: 'SVG',
+    SVG: 'svg',
-    WAV: 'WAV'
+    WAV: 'wav'
 };
 
 module.exports = DataFormat;

src/ScratchStorage.js

@@ -1,4 +1,3 @@
-const Asset = require('./Asset');
 const BuiltinHelper = require('./BuiltinHelper');
 const LocalHelper = require('./LocalHelper');
 const WebHelper = require('./WebHelper');
@@ -17,13 +16,10 @@ class ScratchStorage {
     /**
      * Register a web-based source for assets. Sources will be checked in order of registration.
      * @param {Array.<AssetType>} types - The types of asset provided by this source.
-     * @param {string} urlPattern - The URL pattern to use when loading assets from this source.
+     * @param {UrlFunction} urlFunction - A function which computes a URL from an Asset.
-     * The following sub-strings, if found in the URL pattern, will be replaced with properties of the requested asset:
-     * - '{ID}' will be replaced with the ID of the requested asset.
-     * - '{EXT}' will be replaced with the file extension of the requested asset (such as 'svg').
      */
-    addWebSource (types, urlPattern) {
+    addWebSource (types, urlFunction) {
-        this.webHelper.addSource(types, urlPattern);
+        this.webHelper.addSource(types, urlFunction);
     }
 
     /**
@@ -86,10 +82,4 @@ class ScratchStorage {
     }
 }
 
-/**
- * Export the Asset class so that clients can use it for asset-save requests.
- * @type {Asset}
- */
-ScratchStorage.Asset = Asset;
-
 module.exports = ScratchStorage;

src/WebHelper.js

@@ -1,10 +1,12 @@
+const got = require('got');
+
 const Asset = require('./Asset');
 const Helper = require('./Helper');
-const xhr = require('xhr');
 
 /**
  * @typedef {function} UrlFunction - A function which computes a URL from asset information.
  * @param {Asset} - The asset for which the URL should be computed.
+ * @returns {string} - The URL for the asset.
  */
 
 class WebHelper extends Helper {
@@ -53,8 +55,9 @@ class WebHelper extends Helper {
                 /** @type {UrlFunction} */
                 let urlFunction;
 
-                for (; sourceIndex < sources.length; ++sourceIndex) {
+                while (sourceIndex < sources.length) {
                     const source = sources[sourceIndex];
+                    ++sourceIndex;
                     if (source.types.indexOf(assetType) >= 0) {
                         urlFunction = source.urlFunction;
                         break;
@@ -62,21 +65,28 @@ class WebHelper extends Helper {
                 }
 
                 if (urlFunction) {
+                    const options = {
+                        encoding: null // return body as Buffer
+                    };
                     const url = urlFunction(asset);
-                    xhr({
+                    got(url, options).then(
-                        uri: url
+                        response => {
-                    }, (error, response, body) => {
+                            if (response.status < 200 || response.status >= 300) {
-                        if (error) {
+                                errors.push({url: url, result: response});
+                                tryNextSource();
+                            } else {
+                                /** @type {Buffer} */
+                                const buffer = response.body;
+
+                                // Convert from Buffer to Uint8Array, assuming Node 4.x+ or compatible Webpack
+                                asset.data = new Uint8Array(buffer.buffer);
+                                fulfill(asset);
+                            }
+                        },
+                        error => {
                             errors.push({url: url, result: error});
                             tryNextSource();
-                        } else if (response.status < 200 || response.status >= 300) {
+                        });
-                            errors.push({url: url, result: response});
-                            tryNextSource();
-                        } else {
-                            asset.data = body;
-                            fulfill(asset);
-                        }
-                    });
                 } else if (errors.length > 0) {
                     reject(errors);
                 } else {

src/index.js

@@ -1,7 +1,12 @@
+const Asset = require('./Asset');
+const AssetType = require('./AssetType');
 const ScratchStorage = require('./ScratchStorage');
 
 /**
  * Export for use with NPM & Node.js.
  * @type {ScratchStorage}
  */
-module.exports = ScratchStorage;
+module.exports = Object.assign(ScratchStorage, {
+    Asset: Asset,
+    AssetType: AssetType
+});

test/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+    extends: ['scratch/node', 'scratch/es6']
+};

test/integration/download-known-assets.js

@@ -0,0 +1,94 @@
+const crypto = require('crypto');
+const test = require('tap').test;
+
+const ScratchStorage = require('../../dist/node/scratch-storage');
+const {Asset, AssetType} = ScratchStorage;
+
+/**
+ *
+ * @type {AssetTestInfo[]}
+ * @typedef {object} AssetTestInfo
+ * @property {AssetType} type - The type of the asset.
+ * @property {string} id - The asset's unique ID.
+ */
+const testAssets = [
+    {
+        type: AssetType.Project,
+        id: '117504922',
+        md5: null // don't check MD5 for project without revision ID
+    },
+    {
+        type: AssetType.Project,
+        id: '117504922.d6ae1ffb76f2bc83421cd3f40fc4fd57',
+        md5: '1225460702e149727de28bff4cfd9e23'
+    },
+    {
+        type: AssetType.ImageVector,
+        id: 'f88bf1935daea28f8ca098462a31dbb0', // cat1-a
+        md5: 'f88bf1935daea28f8ca098462a31dbb0'
+    },
+    {
+        type: AssetType.ImageBitmap,
+        id: '7e24c99c1b853e52f8e7f9004416fa34', // squirrel
+        md5: '7e24c99c1b853e52f8e7f9004416fa34'
+    },
+    {
+        type: AssetType.Sound,
+        id: '83c36d806dc92327b9e7049a565c6bff', // meow
+        md5: '83c36d806dc92327b9e7049a565c6bff' // wat
+    }
+];
+
+let storage;
+test('constructor', t => {
+    storage = new ScratchStorage();
+    t.type(storage, ScratchStorage);
+    t.end();
+});
+
+test('addWebSource', t => {
+    t.doesNotThrow(() => {
+        storage.addWebSource(
+            [AssetType.Project],
+            asset => {
+                const [projectId, revision] = asset.assetId.split('.');
+                return revision ?
+                    `https://cdn.projects.scratch.mit.edu/internalapi/project/${projectId}/get/${revision}` :
+                    `https://cdn.projects.scratch.mit.edu/internalapi/project/${projectId}/get/`;
+            });
+    });
+    t.doesNotThrow(() => {
+        storage.addWebSource(
+            [AssetType.ImageVector, AssetType.ImageBitmap, AssetType.Sound],
+            asset => `https://cdn.assets.scratch.mit.edu/internalapi/asset/${asset.assetId}.${asset.assetType.runtimeFormat}/get/`
+        );
+    });
+    t.end();
+});
+
+test('load', t => {
+    const promises = [];
+    for (let i = 0; i < testAssets.length; ++i) {
+        const assetInfo = testAssets[i];
+
+        const promise = storage.load(assetInfo.type, assetInfo.id);
+        t.type(promise, 'Promise');
+
+        promises.push(promise);
+
+        promise.then(asset => {
+            t.type(asset, Asset);
+            t.strictEqual(asset.assetId, assetInfo.id);
+            t.strictEqual(asset.assetType, assetInfo.type);
+            t.ok(asset.data.length);
+
+            if (assetInfo.md5) {
+                const hash = crypto.createHash('md5');
+                hash.update(asset.data);
+                t.strictEqual(hash.digest('hex'), assetInfo.md5);
+            }
+        });
+    }
+
+    return Promise.all(promises);
+});

test/unit/load-default-assets.js

@@ -0,0 +1,51 @@
+const crypto = require('crypto');
+const test = require('tap').test;
+
+const ScratchStorage = require('../../dist/node/scratch-storage');
+const {Asset, AssetType} = ScratchStorage;
+
+const defaultAssetTypes = [AssetType.ImageBitmap, AssetType.ImageVector, AssetType.Sound];
+const defaultIds = {};
+
+let storage;
+test('constructor', t => {
+    storage = new ScratchStorage();
+    t.type(storage, ScratchStorage);
+    t.end();
+});
+
+test('getDefaultAssetId', t => {
+    for (let i = 0; i < defaultAssetTypes.length; ++i) {
+        const assetType = defaultAssetTypes[i];
+        const id = storage.getDefaultAssetId(assetType);
+        t.type(id, 'string');
+        defaultIds[assetType.name] = id;
+    }
+    t.end();
+});
+
+test('load', t => {
+    const promises = [];
+    for (let i = 0; i < defaultAssetTypes.length; ++i) {
+        const assetType = defaultAssetTypes[i];
+        const id = defaultIds[assetType.name];
+
+        const promise = storage.load(assetType, id);
+        t.type(promise, 'Promise');
+
+        promises.push(promise);
+
+        promise.then(asset => {
+            t.type(asset, Asset);
+            t.strictEqual(asset.assetId, id);
+            t.strictEqual(asset.assetType, assetType);
+            t.ok(asset.data.length);
+
+            const hash = crypto.createHash('md5');
+            hash.update(asset.data);
+            t.strictEqual(hash.digest('hex'), id);
+        });
+    }
+
+    return Promise.all(promises);
+});

webpack.config.js

@@ -13,6 +13,10 @@ const base = {
                 query: {
                     presets: ['es2015']
                 }
+            },
+            {
+                test: /\.json$/,
+                loader: 'json-loader'
             }
         ]
     },