Merge pull request #138 from cwillisf/webpack-build-gui

Use scratch-gui source instead of its build output

electron-webpack.json5

@@ -1,9 +1,9 @@
 {
     "main": {
-        "webpackConfig": "webpack.main.additions.js"
+        "webpackConfig": "webpack.main.js"
     },
     "renderer": {
         "template": "src/renderer/index.html",
-        "webpackConfig": "webpack.renderer.additions.js"
+        "webpackConfig": "webpack.renderer.js"
     }
 }

package-deps.json

@@ -0,0 +1,23 @@
+{
+  "description": "Deps of scratch-desktop that don't match scratch-gui. See scripts/rebuild-deps.js",
+  "devDependencies": {
+    "async": "^3.2.0",
+    "babel-plugin-react-intl": "^7.5.7",
+    "copy-webpack-plugin": "^5.1.1",
+    "electron": "^8.2.5",
+    "electron-builder": "^22.6.0",
+    "electron-devtools-installer": "^3.0.0",
+    "electron-notarize": "^0.3.0",
+    "electron-store": "^5.1.1",
+    "electron-webpack": "^2.8.2",
+    "eslint": "^7.0.0",
+    "eslint-config-scratch": "^6.0.0",
+    "eslint-plugin-import": "^2.20.0",
+    "eslint-plugin-react": "^7.20.0",
+    "mkdirp": "^1.0.4",
+    "nets": "^3.2.0",
+    "rimraf": "^3.0.2",
+    "scratch-gui": "github:LLK/scratch-gui#scratch-desktop",
+    "uuid": "^8.0.0"
+  }
+}

package.json

@@ -6,9 +6,10 @@
   "version": "3.14.0",
   "license": "BSD-3-Clause",
   "scripts": {
-    "clean": "rimraf ./dist ./static/assets && node ./scripts/npm-in-gui.js run clean",
+    "clean": "rimraf ./dist ./static/assets",
-    "start": "electron-webpack dev --bail --display-error-details --env.minify=false",
+    "start": "mkdirp ./dist && electron-webpack dev --bail --display-error-details --env.minify=false --no-progress",
-    "compile": "npm run build-gui && electron-webpack --bail --display-error-details --env.minify=false",
+    "postinstall": "npm run rebuild-deps",
+    "compile": "mkdirp ./dist && electron-webpack --bail --display-error-details --env.minify=false --no-progress",
     "fetch": "rimraf ./static/assets/ && mkdirp ./static/assets/ && node ./scripts/fetchMediaLibraryAssets.js",
     "build": "npm run build:dev",
     "build:dev": "npm run compile && npm run doBuild -- --mode=dev",
@@ -19,9 +20,7 @@
     "distDev": "npm run clean && npm run compile && npm run fetch && npm run doBuild -- --mode=dev",
     "test": "npm run test:lint",
     "test:lint": "eslint --cache --color --ext .jsx,.js .",
-    "postinstall": "node ./scripts/npm-in-gui.js install",
+    "rebuild-deps": "node ./scripts/rebuild-deps.js"
-    "build-gui": "node ./scripts/npm-in-gui.js run build",
-    "watch-gui": "node ./scripts/npm-in-gui.js run watch"
   },
   "repository": {
     "type": "git",
@@ -31,43 +30,116 @@
     "source-map-support": "^0.5.19"
   },
   "devDependencies": {
-    "@babel/core": "^7.9.6",
+    "@babel/cli": "^7.1.2",
-    "@babel/plugin-proposal-object-rest-spread": "^7.9.6",
+    "@babel/core": "^7.1.2",
-    "@babel/plugin-syntax-dynamic-import": "^7.8.3",
+    "@babel/plugin-proposal-object-rest-spread": "^7.0.0",
-    "@babel/plugin-transform-async-to-generator": "^7.8.3",
+    "@babel/plugin-syntax-dynamic-import": "^7.0.0",
-    "@babel/preset-env": "^7.9.6",
+    "@babel/plugin-transform-async-to-generator": "^7.1.0",
-    "@babel/preset-react": "^7.9.4",
+    "@babel/preset-env": "^7.1.0",
+    "@babel/preset-react": "^7.0.0",
+    "arraybuffer-loader": "^1.0.6",
     "async": "^3.2.0",
-    "babel-eslint": "^10.1.0",
+    "autoprefixer": "^9.0.1",
-    "babel-loader": "^8.1.0",
+    "babel-core": "7.0.0-bridge.0",
+    "babel-eslint": "^10.0.1",
+    "babel-loader": "^8.0.4",
     "babel-plugin-react-intl": "^7.5.7",
+    "base64-loader": "1.0.0",
+    "bowser": "1.9.4",
+    "chromedriver": "83.0.0",
+    "classnames": "2.2.6",
+    "computed-style-to-inline-style": "3.0.0",
     "copy-webpack-plugin": "^5.1.1",
+    "core-js": "2.5.7",
+    "css-loader": "^1.0.0",
     "electron": "^8.2.5",
     "electron-builder": "^22.6.0",
     "electron-devtools-installer": "^3.0.0",
     "electron-notarize": "^0.3.0",
     "electron-store": "^5.1.1",
     "electron-webpack": "^2.8.2",
+    "enzyme": "^3.5.0",
+    "enzyme-adapter-react-16": "1.3.0",
+    "es6-object-assign": "1.1.0",
     "eslint": "^7.0.0",
     "eslint-config-scratch": "^6.0.0",
+    "eslint-import-resolver-webpack": "^0.11.1",
     "eslint-plugin-import": "^2.20.0",
+    "eslint-plugin-jest": "^22.14.1",
     "eslint-plugin-react": "^7.20.0",
+    "file-loader": "2.0.0",
+    "get-float-time-domain-data": "0.1.0",
+    "get-user-media-promise": "1.1.4",
+    "gh-pages": "github:rschamp/gh-pages#publish-branch-to-subfolder",
+    "html-webpack-plugin": "^3.2.0",
+    "immutable": "3.8.2",
     "intl": "1.2.5",
-    "lodash.bindall": "^4.4.0",
+    "jest": "^21.0.0",
-    "lodash.defaultsdeep": "^4.6.1",
+    "jest-junit": "^7.0.0",
-    "minilog": "^3.1.0",
+    "js-base64": "2.4.9",
+    "keymirror": "0.1.1",
+    "lodash.bindall": "4.4.0",
+    "lodash.debounce": "4.0.8",
+    "lodash.defaultsdeep": "4.6.0",
+    "lodash.isequal": "4.5.0",
+    "lodash.omit": "4.5.0",
+    "lodash.pick": "4.4.0",
+    "lodash.throttle": "4.0.1",
+    "minilog": "3.1.0",
     "mkdirp": "^1.0.4",
     "nets": "^3.2.0",
+    "omggif": "1.0.9",
+    "papaparse": "5.1.1",
+    "postcss-import": "^12.0.0",
+    "postcss-loader": "^3.0.0",
+    "postcss-simple-vars": "^5.0.1",
+    "prop-types": "^15.5.10",
+    "query-string": "^5.1.1",
+    "raf": "^3.4.0",
+    "raw-loader": "^0.5.1",
     "react": "16.2.0",
-    "react-dom": "16.2.1",
+    "react-contextmenu": "2.9.4",
-    "react-intl": "2.8.0",
+    "react-dom": "16.2.0",
+    "react-draggable": "3.0.5",
+    "react-ga": "2.5.3",
+    "react-intl": "2.9.0",
+    "react-modal": "3.9.1",
+    "react-popover": "0.5.10",
     "react-redux": "5.0.7",
-    "redux": "3.5.2",
+    "react-responsive": "5.0.0",
+    "react-style-proptype": "3.2.2",
+    "react-tabs": "2.3.0",
+    "react-test-renderer": "16.2.0",
+    "react-tooltip": "3.8.0",
+    "react-virtualized": "9.20.1",
+    "react-visibility-sensor": "5.0.2",
+    "redux": "3.7.2",
+    "redux-mock-store": "^1.2.3",
+    "redux-throttle": "0.1.1",
     "rimraf": "^3.0.2",
+    "scratch-audio": "0.1.0-prerelease.20200528195344",
+    "scratch-blocks": "0.1.0-prerelease.20200721135911",
     "scratch-gui": "github:LLK/scratch-gui#scratch-desktop",
-    "source-map-loader": "^0.2.4",
+    "scratch-l10n": "3.10.20200722030735",
+    "scratch-paint": "0.2.0-prerelease.20200721201211",
+    "scratch-render": "0.1.0-prerelease.20200721190141",
+    "scratch-storage": "1.3.3",
+    "scratch-svg-renderer": "0.2.0-prerelease.20200610220938",
+    "scratch-vm": "0.2.0-prerelease.20200720182258",
+    "selenium-webdriver": "3.6.0",
+    "startaudiocontext": "1.2.1",
+    "style-loader": "^0.23.0",
+    "svg-to-image": "1.1.3",
+    "text-encoding": "0.7.0",
+    "to-style": "1.3.3",
+    "uglifyjs-webpack-plugin": "^1.2.5",
     "uuid": "^8.0.0",
-    "webpack": "^4.43.0"
+    "wav-encoder": "1.3.0",
+    "web-audio-test-api": "^0.5.2",
+    "webpack": "^4.6.0",
+    "webpack-cli": "^3.1.0",
+    "webpack-dev-server": "^3.1.3",
+    "xhr": "2.5.0"
   },
   "resolutions": {
     "upath": "^1.0.5"

scripts/lib/libraries.js

@@ -1,7 +1,7 @@
-const backdrops = require('scratch-gui/dist/libraries/backdrops.json');
+const backdrops = require('scratch-gui/src/lib/libraries/backdrops.json');
-const costumes = require('scratch-gui/dist/libraries/costumes.json');
+const costumes = require('scratch-gui/src/lib/libraries/costumes.json');
-const sounds = require('scratch-gui/dist/libraries/sounds.json');
+const sounds = require('scratch-gui/src/lib/libraries/sounds.json');
-const sprites = require('scratch-gui/dist/libraries/sprites.json');
+const sprites = require('scratch-gui/src/lib/libraries/sprites.json');
 
 const libraries = {
     backdrops,

scripts/npm-in-gui.js

@@ -1,25 +0,0 @@
-const childProcess = require('child_process');
-const path = require('path');
-
-// process.argv: ['node', 'run-in-gui.js', 'foo', ...]
-const args = process.argv.slice(2);
-
-// copy environment (including PATH) then add or replace BUILD_MODE and STATIC_PATH
-const env = {
-    ...process.env,
-    BUILD_MODE: 'dist',
-    STATIC_PATH: 'static'
-};
-
-const child = childProcess.spawnSync(
-    'npm', args,
-    {
-        cwd: path.join('node_modules', 'scratch-gui'),
-        env,
-        shell: true,
-        stdio: 'inherit'
-    }
-);
-
-if (child.error) throw child.error;
-process.exit(child.status);

scripts/rebuild-deps.js

@@ -0,0 +1,70 @@
+const fs = require('fs');
+
+const desktopPackage = require('../package.json');
+const desktopRawPackage = require('../package-deps.json');
+const guiPackage = require('scratch-gui/package.json');
+
+/**
+ * Sort an object's keys.
+ * @param {object} obj - the object whose keys should be sorted.
+ * @returns {object} - a new object with the same keys and values, but the keys are sorted by {@link Array#sort}.
+ */
+const sortObject = obj => Object
+    .keys(obj)
+    .sort()
+    .reduce(
+        (accumulator, currentKey) => {
+            accumulator[currentKey] = obj[currentKey];
+            return accumulator;
+        },
+        {}
+    );
+
+/**
+ * Make a shallow comparison of two objects, returning true if they match. This shallow comparison returns true iff:
+ * - the two objects have the same number of keys, AND
+ * - each key of obj1 is present in obj2 (via hasOwnProperty), AND
+ * - the value associated with each key in obj1 matches (using ===) the value associated with the same key in obj2
+ * Note that the order of keys is not compared.
+ * @param {object} obj1 - the first object to compare
+ * @param {object} obj2 - the second object to compare
+ * @returns {boolean} - true if the objects are shallow-identical.
+ */
+const shallowCompare = (obj1, obj2) =>
+    Object.keys(obj1).length === Object.keys(obj2).length &&
+    Object.keys(obj1).every(k => Object.prototype.hasOwnProperty.call(obj2, k) && (obj1[k] === obj2[k]));
+
+/**
+ * Merge `scratch-gui` and `scratch-desktop` dependencies:
+ * 1. Merge the dependencies found in `scratch-gui/package.json` with those in `package-deps.json`.
+ * 2. Report statistics.
+ * 3. If the result differs from `package.json`, rewrite `package.json` with the new dependencies and print a warning.
+ */
+const rebuildDeps = () => {
+    const oldDevDependencies = desktopPackage.devDependencies;
+    desktopPackage.devDependencies = sortObject(Object.assign(
+        {},
+        guiPackage.devDependencies,
+        desktopRawPackage.devDependencies
+    ));
+
+    console.log(`Direct dependencies (desktop / gui / merged): ${
+        Object.keys(desktopRawPackage.devDependencies).length} / ${
+        Object.keys(guiPackage.devDependencies).length} / ${
+        Object.keys(desktopPackage.devDependencies).length}`);
+    if (shallowCompare(oldDevDependencies, desktopPackage.devDependencies)) {
+        console.log('Dependencies in package.json were already up-to-date.');
+    } else {
+        let newDesktopPackage = JSON.stringify(desktopPackage, null, 2);
+        newDesktopPackage += '\n';
+        fs.writeFileSync('package.json', newDesktopPackage);
+        console.warn(
+            '\n', // NPM adds a prefix to all BUT the first line, so this makes the remaining lines format correctly
+            '***\n',
+            '*** Dependencies in package.json have been updated! Please re-run npm to install these updates.\n',
+            '***\n'
+        );
+    }
+};
+
+rebuildDeps();

webpack.main.additions.js

@@ -1,11 +0,0 @@
-const path = require('path');
-
-const makeConfig = require('./webpack.makeConfig.js');
-
-module.exports = makeConfig({
-    name: 'main',
-    useReact: false,
-    babelPaths: [
-        path.resolve(__dirname, 'src', 'main')
-    ]
-});

webpack.main.js

@@ -0,0 +1,16 @@
+const path = require('path');
+
+const makeConfig = require('./webpack.makeConfig.js');
+
+module.exports = defaultConfig =>
+    makeConfig(
+        defaultConfig,
+        {
+            name: 'main',
+            useReact: false,
+            disableDefaultRulesForExtensions: ['js'],
+            babelPaths: [
+                path.resolve(__dirname, 'src', 'main')
+            ]
+        }
+    );

webpack.makeConfig.js

@@ -1,17 +1,23 @@
 const childProcess = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const util = require('util');
 
 const electronPath = require('electron');
 const webpack = require('webpack');
+const merge = require('webpack-merge');
+
+// PostCss
+const autoprefixer = require('autoprefixer');
+const postcssVars = require('postcss-simple-vars');
+const postcssImport = require('postcss-import');
 
 const isProduction = (process.env.NODE_ENV === 'production');
 
 const electronVersion = childProcess.execSync(`${electronPath} --version`, {encoding: 'utf8'}).trim();
 console.log(`Targeting Electron ${electronVersion}`); // eslint-disable-line no-console
 
-const makeConfig = function (options) {
+const makeConfig = function (defaultConfig, options) {
-    // eslint-disable-next-line no-console
-    console.log(`Module "${options.name}" building in production mode? ${isProduction}`);
-
     const babelOptions = {
         // Explicitly disable babelrc so we don't catch various config in much lower dependencies.
         babelrc: false,
@@ -33,30 +39,68 @@ const makeConfig = function (options) {
         }]);
     }
 
-    return {
+    // TODO: consider adjusting these rules instead of discarding them in at least some cases
+    if (options.disableDefaultRulesForExtensions) {
+        defaultConfig.module.rules = defaultConfig.module.rules.filter(rule => {
+            if (!(rule.test instanceof RegExp)) {
+                // currently we don't support overriding other kinds of rules
+                return true;
+            }
+            // disable default rules for any file extension listed here
+            // we will handle these files in some other way (see below)
+            // OR we want to avoid any processing at all (such as with fonts)
+            const shouldDisable = options.disableDefaultRulesForExtensions.some(
+                ext => rule.test.test(`test.${ext}`)
+            );
+            const statusWord = shouldDisable ? 'Discarding' : 'Keeping';
+            console.log(`${options.name}: ${statusWord} electron-webpack default rule for ${rule.test}`);
+            return !shouldDisable;
+        });
+    }
+
+    const config = merge.smart(defaultConfig, {
         devtool: 'cheap-module-eval-source-map',
         mode: isProduction ? 'production' : 'development',
         module: {
             rules: [
-                // Override the *.js defaults from electron-webpack
-                // The test/include/exclude must match the defaults exactly for webpack-merge to do the override
-                {
-                    test: /\.js$/,
-                    exclude: /(node_modules|bower_components)/,
-                    loader: 'babel-loader',
-                    options: babelOptions
-                },
-                // Add a new rule for the other files we want to run through babel
                 {
                     test: sourceFileTest,
                     include: options.babelPaths,
                     loader: 'babel-loader',
                     options: babelOptions
                 },
+                { // coped from scratch-gui
+                    test: /\.css$/,
+                    use: [{
+                        loader: 'style-loader'
+                    }, {
+                        loader: 'css-loader',
+                        options: {
+                            modules: true,
+                            importLoaders: 1,
+                            localIdentName: '[name]_[local]_[hash:base64:5]',
+                            camelCase: true
+                        }
+                    }, {
+                        loader: 'postcss-loader',
+                        options: {
+                            ident: 'postcss',
+                            plugins: function () {
+                                return [
+                                    postcssImport,
+                                    postcssVars,
+                                    autoprefixer
+                                ];
+                            }
+                        }
+                    }]
+                },
                 {
-                    test: sourceFileTest,
+                    test: /\.(svg|png|wav|gif|jpg)$/,
-                    loader: 'source-map-loader',
+                    loader: 'file-loader',
-                    enforce: 'pre'
+                    options: {
+                        outputPath: 'static/assets/'
+                    }
                 }
             ]
         },
@@ -67,9 +111,27 @@ const makeConfig = function (options) {
         ].concat(options.plugins || []),
         resolve: {
             cacheWithContext: false,
-            symlinks: false
+            symlinks: false,
+            alias: {
+                // act like scratch-gui has this line in its package.json:
+                //   "browser": "./src/index.js"
+                'scratch-gui$': path.resolve(__dirname, 'node_modules', 'scratch-gui', 'src', 'index.js')
             }
-    };
+        }
+    });
+
+    // If we're not on CI, enable Webpack progress output
+    // Note that electron-webpack enables this by default, so use '--no-progress' to avoid double-adding this plugin
+    if (!process.env.CI) {
+        config.plugins.push(new webpack.ProgressPlugin());
+    }
+
+    fs.writeFileSync(
+        `dist/webpack.${options.name}.js`,
+        `module.exports = ${util.inspect(config, {depth: null})};\n`
+    );
+
+    return config;
 };
 
 module.exports = makeConfig;

webpack.renderer.additions.js

@@ -1,19 +0,0 @@
-const path = require('path');
-
-const CopyWebpackPlugin = require('copy-webpack-plugin');
-
-const makeConfig = require('./webpack.makeConfig.js');
-
-module.exports = makeConfig({
-    name: 'renderer',
-    useReact: true,
-    babelPaths: [
-        path.resolve(__dirname, 'src', 'renderer')
-    ],
-    plugins: [
-        new CopyWebpackPlugin([{
-            from: path.resolve(__dirname, 'node_modules', 'scratch-gui', 'dist', 'static'),
-            to: 'static'
-        }])
-    ]
-});

webpack.renderer.js

@@ -0,0 +1,36 @@
+const path = require('path');
+
+const CopyWebpackPlugin = require('copy-webpack-plugin');
+
+const makeConfig = require('./webpack.makeConfig.js');
+
+module.exports = defaultConfig =>
+    makeConfig(
+        defaultConfig,
+        {
+            name: 'renderer',
+            useReact: true,
+            disableDefaultRulesForExtensions: ['js', 'jsx', 'css', 'svg', 'png', 'wav', 'gif', 'jpg', 'ttf'],
+            babelPaths: [
+                path.resolve(__dirname, 'src', 'renderer'),
+                /node_modules[\\/]+scratch-[^\\/]+[\\/]+src/,
+                /node_modules[\\/]+pify/,
+                /node_modules[\\/]+@vernier[\\/]+godirect/
+            ],
+            plugins: [
+                new CopyWebpackPlugin([{
+                    from: 'node_modules/scratch-blocks/media',
+                    to: 'static/blocks-media'
+                }]),
+                new CopyWebpackPlugin([{
+                    from: 'extension-worker.{js,js.map}',
+                    context: 'node_modules/scratch-vm/dist/web'
+                }]),
+                new CopyWebpackPlugin([{
+                    from: 'src/lib/libraries/*.json',
+                    to: 'static/libraries',
+                    flatten: true
+                }])
+            ]
+        }
+    );