mirror of
https://github.com/scratchfoundation/scratch-desktop.git
synced 2025-07-06 01:20:39 -04:00
Add media library assets, fetch script
This commit is contained in:
parent
146a6f0de0
commit
42cba6414e
8 changed files with 155 additions and 10 deletions
|
@ -3,5 +3,5 @@ module.exports = {
|
||||||
env: {
|
env: {
|
||||||
node: true
|
node: true
|
||||||
},
|
},
|
||||||
extends: ['scratch', 'scratch/node']
|
extends: ['scratch', 'scratch/es6', 'scratch/node']
|
||||||
};
|
};
|
||||||
|
|
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -4,6 +4,9 @@ dist/
|
||||||
node_modules/
|
node_modules/
|
||||||
thumbs.db
|
thumbs.db
|
||||||
|
|
||||||
|
# don't store the assets downloaded with the `fetch` script
|
||||||
|
/src/static/assets/
|
||||||
|
|
||||||
# generated translation files
|
# generated translation files
|
||||||
/translations
|
/translations
|
||||||
/locale
|
/locale
|
||||||
|
|
19
package-lock.json
generated
19
package-lock.json
generated
|
@ -1549,10 +1549,13 @@
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"async": {
|
"async": {
|
||||||
"version": "1.5.2",
|
"version": "2.6.1",
|
||||||
"resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
|
"resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz",
|
||||||
"integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
|
"integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==",
|
||||||
"dev": true
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"lodash": "4.17.11"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"async-each": {
|
"async-each": {
|
||||||
"version": "1.0.1",
|
"version": "1.0.1",
|
||||||
|
@ -4391,7 +4394,7 @@
|
||||||
},
|
},
|
||||||
"events": {
|
"events": {
|
||||||
"version": "1.1.1",
|
"version": "1.1.1",
|
||||||
"resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
|
||||||
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=",
|
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=",
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
|
@ -8088,6 +8091,12 @@
|
||||||
"mkdirp": "0.5.1"
|
"mkdirp": "0.5.1"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"async": {
|
||||||
|
"version": "1.5.2",
|
||||||
|
"resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz",
|
||||||
|
"integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
"debug": {
|
"debug": {
|
||||||
"version": "2.6.9",
|
"version": "2.6.9",
|
||||||
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "electron-webpack dev --bail --display-error-details --env.minify=false",
|
"start": "electron-webpack dev --bail --display-error-details --env.minify=false",
|
||||||
"compile": "electron-webpack --bail --display-error-details --env.minify=false",
|
"compile": "electron-webpack --bail --display-error-details --env.minify=false",
|
||||||
|
"fetch": "node ./scripts/fetchMediaLibraryAssets.js",
|
||||||
"dist": "npm run compile && electron-builder",
|
"dist": "npm run compile && electron-builder",
|
||||||
"dist:dir": "npm run dist -- --dir -c.compression=store -c.mac.identity=null",
|
"dist:dir": "npm run dist -- --dir -c.compression=store -c.mac.identity=null",
|
||||||
"lint": "eslint --cache --color --ext .jsx,.js ."
|
"lint": "eslint --cache --color --ext .jsx,.js ."
|
||||||
|
@ -15,6 +16,7 @@
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/plugin-proposal-object-rest-spread": "^7.0.0",
|
"@babel/plugin-proposal-object-rest-spread": "^7.0.0",
|
||||||
"@babel/preset-react": "^7.0.0",
|
"@babel/preset-react": "^7.0.0",
|
||||||
|
"async": "^2.6.1",
|
||||||
"babel-eslint": "^10.0.0",
|
"babel-eslint": "^10.0.0",
|
||||||
"babel-plugin-react-intl": "^3.0.1",
|
"babel-plugin-react-intl": "^3.0.1",
|
||||||
"copy-webpack-plugin": "^4.5.2",
|
"copy-webpack-plugin": "^4.5.2",
|
||||||
|
|
5
scripts/.eslintrc.js
Normal file
5
scripts/.eslintrc.js
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
module.exports = {
|
||||||
|
rules: {
|
||||||
|
'no-console': 'off'
|
||||||
|
}
|
||||||
|
};
|
107
scripts/fetchMediaLibraryAssets.js
Normal file
107
scripts/fetchMediaLibraryAssets.js
Normal file
|
@ -0,0 +1,107 @@
|
||||||
|
const fs = require('fs');
|
||||||
|
const https = require('https');
|
||||||
|
const path = require('path');
|
||||||
|
const util = require('util');
|
||||||
|
|
||||||
|
const async = require('async');
|
||||||
|
|
||||||
|
const libraries = require('./lib/libraries');
|
||||||
|
|
||||||
|
const ASSET_HOST = 'cdn.assets.scratch.mit.edu';
|
||||||
|
const NUM_SIMULTANEOUS_DOWNLOADS = 5;
|
||||||
|
|
||||||
|
|
||||||
|
const describe = function (object) {
|
||||||
|
return util.inspect(object, false, Infinity, true);
|
||||||
|
};
|
||||||
|
|
||||||
|
const collectSimple = function (library, debugLabel = 'Item', dest = new Set()) {
|
||||||
|
library.forEach(item => {
|
||||||
|
let md5Count = 0;
|
||||||
|
if (item.md5) {
|
||||||
|
++md5Count;
|
||||||
|
dest.add(item.md5);
|
||||||
|
}
|
||||||
|
if (item.baseLayerMD5) {
|
||||||
|
++md5Count;
|
||||||
|
dest.add(item.baseLayerMD5);
|
||||||
|
}
|
||||||
|
if (md5Count < 1) {
|
||||||
|
console.warn(`${debugLabel} has no MD5 property:\n${describe(item)}`);
|
||||||
|
} else if (md5Count > 1) {
|
||||||
|
// is this actually bad?
|
||||||
|
console.warn(`${debugLabel} has multiple MD5 properties:\n${describe(item)}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return dest;
|
||||||
|
};
|
||||||
|
|
||||||
|
const collectAssets = function (dest = new Set()) {
|
||||||
|
collectSimple(libraries.backdrops, 'Backdrop', dest);
|
||||||
|
collectSimple(libraries.costumes, 'Costume', dest);
|
||||||
|
collectSimple(libraries.sounds, 'Sound', dest);
|
||||||
|
libraries.sprites.forEach(sprite => {
|
||||||
|
if (sprite.md5) {
|
||||||
|
dest.add(sprite.md5);
|
||||||
|
} else {
|
||||||
|
console.warn(`Sprite has no MD5 property:\n${describe(sprite)}`);
|
||||||
|
}
|
||||||
|
if (sprite.json.costumes) {
|
||||||
|
collectSimple(sprite.json.costumes, `Costume for sprite ${sprite.name}`);
|
||||||
|
}
|
||||||
|
if (sprite.json.sounds) {
|
||||||
|
collectSimple(sprite.json.sounds, `Sound for sprite ${sprite.name}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return dest;
|
||||||
|
};
|
||||||
|
|
||||||
|
const connectionPool = [];
|
||||||
|
|
||||||
|
const fetchAsset = function (md5, callback) {
|
||||||
|
const myAgent = connectionPool.pop() || new https.Agent({keepAlive: true});
|
||||||
|
const getOptions = {
|
||||||
|
host: ASSET_HOST,
|
||||||
|
path: `/internalapi/asset/${md5}/get/`,
|
||||||
|
agent: myAgent
|
||||||
|
};
|
||||||
|
const urlHuman = `//${getOptions.host}${getOptions.path}`;
|
||||||
|
https.get(getOptions, response => {
|
||||||
|
if (response.statusCode !== 200) {
|
||||||
|
callback(new Error(`Request failed: status code ${response.statusCode} for ${urlHuman}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const stream = fs.createWriteStream(path.resolve('src', 'static', 'assets', md5), {encoding: 'binary'});
|
||||||
|
stream.on('error', callback);
|
||||||
|
response.on('data', chunk => {
|
||||||
|
stream.write(chunk);
|
||||||
|
});
|
||||||
|
response.on('end', () => {
|
||||||
|
connectionPool.push(myAgent);
|
||||||
|
stream.end();
|
||||||
|
console.log(`Fetched ${urlHuman}`);
|
||||||
|
callback();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const fetchAllAssets = function () {
|
||||||
|
const allAssets = collectAssets();
|
||||||
|
console.log(`Total library assets: ${allAssets.size}`);
|
||||||
|
|
||||||
|
async.forEachLimit(allAssets, NUM_SIMULTANEOUS_DOWNLOADS, fetchAsset, err => {
|
||||||
|
if (err) {
|
||||||
|
console.error(`Fetch failed:\n${describe(err)}`);
|
||||||
|
} else {
|
||||||
|
console.log('Fetch succeeded.');
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Shutting down ${connectionPool.length} agents.`);
|
||||||
|
while (connectionPool.length > 0) {
|
||||||
|
connectionPool.pop().destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
fetchAllAssets();
|
13
scripts/lib/libraries.js
Normal file
13
scripts/lib/libraries.js
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
const backdrops = require('scratch-gui/src/lib/libraries/backdrops.json');
|
||||||
|
const costumes = require('scratch-gui/src/lib/libraries/costumes.json');
|
||||||
|
const sounds = require('scratch-gui/src/lib/libraries/sounds.json');
|
||||||
|
const sprites = require('scratch-gui/src/lib/libraries/sprites.json');
|
||||||
|
|
||||||
|
const libraries = {
|
||||||
|
backdrops,
|
||||||
|
costumes,
|
||||||
|
sounds,
|
||||||
|
sprites
|
||||||
|
};
|
||||||
|
|
||||||
|
module.exports = libraries;
|
|
@ -58,10 +58,16 @@ module.exports = {
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
plugins: [
|
plugins: [
|
||||||
new CopyWebpackPlugin([{
|
new CopyWebpackPlugin([
|
||||||
from: path.resolve(__dirname, 'node_modules', 'scratch-gui', 'dist', 'static'),
|
{
|
||||||
to: 'static'
|
from: path.resolve(__dirname, 'node_modules', 'scratch-gui', 'dist', 'static'),
|
||||||
}])
|
to: 'static'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
from: path.resolve(__dirname, 'src', 'static'),
|
||||||
|
to: 'static'
|
||||||
|
}
|
||||||
|
])
|
||||||
],
|
],
|
||||||
resolve: {
|
resolve: {
|
||||||
symlinks: false
|
symlinks: false
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue