Add media library assets, fetch script

.eslintrc.js

@@ -3,5 +3,5 @@ module.exports = {
     env: {
         node: true
     },
-    extends: ['scratch', 'scratch/node']
+    extends: ['scratch', 'scratch/es6', 'scratch/node']
 };

.gitignore

@@ -4,6 +4,9 @@ dist/
 node_modules/
 thumbs.db
 
+# don't store the assets downloaded with the `fetch` script
+/src/static/assets/
+
 # generated translation files
 /translations
 /locale

package-lock.json

@@ -1549,10 +1549,13 @@
       "dev": true
     },
     "async": {
-      "version": "1.5.2",
-      "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
-      "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
-      "dev": true
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz",
+      "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==",
+      "dev": true,
+      "requires": {
+        "lodash": "4.17.11"
+      }
     },
     "async-each": {
       "version": "1.0.1",
@@ -4391,7 +4394,7 @@
     },
     "events": {
       "version": "1.1.1",
-      "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
       "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=",
       "dev": true
     },
@@ -8088,6 +8091,12 @@
         "mkdirp": "0.5.1"
       },
       "dependencies": {
+        "async": {
+          "version": "1.5.2",
+          "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz",
+          "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
+          "dev": true
+        },
         "debug": {
           "version": "2.6.9",
           "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",

package.json

@@ -5,6 +5,7 @@
   "scripts": {
     "start": "electron-webpack dev --bail --display-error-details --env.minify=false",
     "compile": "electron-webpack --bail --display-error-details --env.minify=false",
+    "fetch": "node ./scripts/fetchMediaLibraryAssets.js",
     "dist": "npm run compile && electron-builder",
     "dist:dir": "npm run dist -- --dir -c.compression=store -c.mac.identity=null",
     "lint": "eslint --cache --color --ext .jsx,.js ."
@@ -15,6 +16,7 @@
   "devDependencies": {
     "@babel/plugin-proposal-object-rest-spread": "^7.0.0",
     "@babel/preset-react": "^7.0.0",
+    "async": "^2.6.1",
     "babel-eslint": "^10.0.0",
     "babel-plugin-react-intl": "^3.0.1",
     "copy-webpack-plugin": "^4.5.2",

scripts/.eslintrc.js

@@ -0,0 +1,5 @@
+module.exports = {
+    rules: {
+        'no-console': 'off'
+    }
+};

scripts/fetchMediaLibraryAssets.js

@@ -0,0 +1,107 @@
+const fs = require('fs');
+const https = require('https');
+const path = require('path');
+const util = require('util');
+
+const async = require('async');
+
+const libraries = require('./lib/libraries');
+
+const ASSET_HOST = 'cdn.assets.scratch.mit.edu';
+const NUM_SIMULTANEOUS_DOWNLOADS = 5;
+
+
+const describe = function (object) {
+    return util.inspect(object, false, Infinity, true);
+};
+
+const collectSimple = function (library, debugLabel = 'Item', dest = new Set()) {
+    library.forEach(item => {
+        let md5Count = 0;
+        if (item.md5) {
+            ++md5Count;
+            dest.add(item.md5);
+        }
+        if (item.baseLayerMD5) {
+            ++md5Count;
+            dest.add(item.baseLayerMD5);
+        }
+        if (md5Count < 1) {
+            console.warn(`${debugLabel} has no MD5 property:\n${describe(item)}`);
+        } else if (md5Count > 1) {
+            // is this actually bad?
+            console.warn(`${debugLabel} has multiple MD5 properties:\n${describe(item)}`);
+        }
+    });
+    return dest;
+};
+
+const collectAssets = function (dest = new Set()) {
+    collectSimple(libraries.backdrops, 'Backdrop', dest);
+    collectSimple(libraries.costumes, 'Costume', dest);
+    collectSimple(libraries.sounds, 'Sound', dest);
+    libraries.sprites.forEach(sprite => {
+        if (sprite.md5) {
+            dest.add(sprite.md5);
+        } else {
+            console.warn(`Sprite has no MD5 property:\n${describe(sprite)}`);
+        }
+        if (sprite.json.costumes) {
+            collectSimple(sprite.json.costumes, `Costume for sprite ${sprite.name}`);
+        }
+        if (sprite.json.sounds) {
+            collectSimple(sprite.json.sounds, `Sound for sprite ${sprite.name}`);
+        }
+    });
+    return dest;
+};
+
+const connectionPool = [];
+
+const fetchAsset = function (md5, callback) {
+    const myAgent = connectionPool.pop() || new https.Agent({keepAlive: true});
+    const getOptions = {
+        host: ASSET_HOST,
+        path: `/internalapi/asset/${md5}/get/`,
+        agent: myAgent
+    };
+    const urlHuman = `//${getOptions.host}${getOptions.path}`;
+    https.get(getOptions, response => {
+        if (response.statusCode !== 200) {
+            callback(new Error(`Request failed: status code ${response.statusCode} for ${urlHuman}`));
+            return;
+        }
+
+        const stream = fs.createWriteStream(path.resolve('src', 'static', 'assets', md5), {encoding: 'binary'});
+        stream.on('error', callback);
+        response.on('data', chunk => {
+            stream.write(chunk);
+        });
+        response.on('end', () => {
+            connectionPool.push(myAgent);
+            stream.end();
+            console.log(`Fetched ${urlHuman}`);
+            callback();
+        });
+    });
+};
+
+const fetchAllAssets = function () {
+    const allAssets = collectAssets();
+    console.log(`Total library assets: ${allAssets.size}`);
+
+    async.forEachLimit(allAssets, NUM_SIMULTANEOUS_DOWNLOADS, fetchAsset, err => {
+        if (err) {
+            console.error(`Fetch failed:\n${describe(err)}`);
+        } else {
+            console.log('Fetch succeeded.');
+        }
+
+        console.log(`Shutting down ${connectionPool.length} agents.`);
+        while (connectionPool.length > 0) {
+            connectionPool.pop().destroy();
+        }
+    });
+};
+
+fetchAllAssets();

scripts/lib/libraries.js

@@ -0,0 +1,13 @@
+const backdrops = require('scratch-gui/src/lib/libraries/backdrops.json');
+const costumes = require('scratch-gui/src/lib/libraries/costumes.json');
+const sounds = require('scratch-gui/src/lib/libraries/sounds.json');
+const sprites = require('scratch-gui/src/lib/libraries/sprites.json');
+
+const libraries = {
+    backdrops,
+    costumes,
+    sounds,
+    sprites
+};
+
+module.exports = libraries;

webpack.renderer.additions.js

@@ -58,10 +58,16 @@ module.exports = {
         }
     },
     plugins: [
-        new CopyWebpackPlugin([{
-            from: path.resolve(__dirname, 'node_modules', 'scratch-gui', 'dist', 'static'),
-            to: 'static'
-        }])
+        new CopyWebpackPlugin([
+            {
+                from: path.resolve(__dirname, 'node_modules', 'scratch-gui', 'dist', 'static'),
+                to: 'static'
+            },
+            {
+                from: path.resolve(__dirname, 'src', 'static'),
+                to: 'static'
+            }
+        ])
     ],
     resolve: {
         symlinks: false