Add scripts for automation

* bumped minor version
* added localeMap for converting between transifex locales and the ones used by Scratch (e.g., pt-br - pt_BR)
* using the transifex node package to automatically sync translations
* added scripts:
  * sync_tx_src: uploads an en.json source file, for use by client packages
  * sync_tx_translations: downloads gui translations, used by this repo, flattens Chrome i18n json into plain key-value json.
  * sync_tx_blocks: same as above, but blocks need slightly different handling
  * validate_translations: check the translation json files for basic translation errors
  * tx_util - methods shared by sync and validate
* simplified build-data because jsons have already been flattened
* added new npm tasks

.babelrc

@@ -1,6 +1,9 @@
 {
     "plugins": [
-        "transform-object-rest-spread"
+        "@babel/plugin-proposal-object-rest-spread"
     ],
-    "presets": [["env", {"targets": {"browsers": ["last 3 versions", "Safari >= 8", "iOS >= 8"]}}]]
+    "presets": [
+        ["@babel/preset-env", {"targets": {"browsers": ["last 3 versions", "Safari >= 8", "iOS >= 8"]}}],
+        "@babel/preset-react"
+    ]
 }

package.json

@@ -1,11 +1,12 @@
 {
   "name": "scratch-l10n",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Localization for the Scratch 3.0 components",
   "main": "./dist/l10n.js",
   "browser": "./src/index.js",
   "bin": {
-    "build-i18n-src": "./scripts/build-i18n-src.js"
+    "build-i18n-src": "./scripts/build-i18n-src.js",
+    "sync-tx-src": "./scripts/build-i18n-src.js"
   },
   "scripts": {
     "build:data": "babel-node scripts/build-data",
@@ -14,7 +15,17 @@
     "lint:js": "eslint . --ext .js",
     "lint:json": "jshint -e .json www editor/blocks editor/extensions editor/interface editor/paint-editor",
     "lint": "npm run lint:js && npm run lint:json",
-    "test": "npm run lint && npm run build"
+    "sync:gui": "babel-node scripts/sync_tx_translations scratch-editor interface ./editor/interface/",
+    "sync:paint": "babel-node scripts/sync_tx_translations scratch-editor paint-editor ./editor/paint-editor/",
+    "sync:extensions": "babel-node scripts/sync_tx_translations scratch-editor extensions ./editor/extensions/",
+    "sync:blocks": "babel-node scripts/sync_tx_blocks ./editor/blocks/",
+    "sync:editor": "npm run sync:blocks && npm run sync:extensions && npm run sync:paint && npm run sync:gui",
+    "test": "npm run lint:js && npm run validate:editor && npm run build && npm run lint:json",
+    "validate:blocks": "babel-node scripts/validate_translations ./editor/blocks/",
+    "validate:extensions": "babel-node scripts/validate_translations ./editor/extensions/",
+    "validate:gui": "babel-node scripts/validate_translations ./editor/interface/",
+    "validate:paint": "babel-node scripts/validate_translations ./editor/paint-editor/",
+    "validate:editor": "npm run validate:blocks && npm run validate:extensions && npm run validate:gui && npm run validate:paint"
   },
   "repository": {
     "type": "git",
@@ -27,26 +38,36 @@
   },
   "homepage": "https://github.com/LLK/scratch-l10n#readme",
   "dependencies": {
-    "babel-cli": "^6.26.0",
+    "@babel/cli": "^7.1.2",
-    "babel-plugin-intl": "^0.1.1",
+    "@babel/core": "^7.1.2",
-    "babel-plugin-react-intl": "^2.3.1",
+    "babel-plugin-react-intl": "^2.3.1"
-    "react-intl": "2.4.0"
   },
   "devDependencies": {
-    "babel-core": "^6.26.0",
+    "@babel/node": "7.0.0",
-    "babel-eslint": "^7.2.3",
+    "@babel/plugin-proposal-object-rest-spread": "^7.0.0",
-    "babel-loader": "^7.1.4",
+    "@babel/plugin-syntax-dynamic-import": "^7.0.0",
-    "babel-plugin-transform-object-rest-spread": "^6.26.0",
+    "@babel/plugin-transform-async-to-generator": "^7.1.0",
-    "babel-preset-env": "^1.6.1",
+    "@babel/preset-env": "^7.1.0",
-    "eslint": "^4.6.1",
+    "@babel/preset-react": "^7.0.0",
-    "eslint-config-scratch": "^4.0.0",
+    "async": "2.6.1",
-    "eslint-plugin-import": "^2.7.0",
+    "babel-eslint": "^10.0.1",
+    "babel-loader": "8.0.4",
+    "eslint": "^5.0.1",
+    "eslint-config-scratch": "^5.0.0",
+    "eslint-plugin-import": "^2.8.0",
+    "eslint-plugin-react": "^7.5.1",
+    "format-message-cli": "6.2.1",
+    "format-message-parse": "6.2.0",
     "jshint": "2.9.5",
     "json": "^9.0.6",
     "jsonlint": "1.6.3",
     "lodash.defaultsdeep": "4.6.0",
     "mkdirp": "^0.5.1",
+    "p-limit": "2.0.0",
+    "p-queue": "3.0.0",
+    "react-intl": "2.4.0",
     "rimraf": "^2.6.2",
+    "transifex": "1.5.0",
     "webpack": "^4.6.0",
     "webpack-cli": "^3.1.2"
   }

scripts/build-data.js

@@ -48,52 +48,22 @@ const MSGS_DIR = './locales/';
 mkdirpSync(MSGS_DIR);
 let missingLocales = [];
 
-// generate messages for gui components - files are Chrome i18n format json
+const combineJson = (component) => {
-let components = ['interface', 'extensions', 'paint-editor'];
+    return Object.keys(locales).reduce((collection, lang) => {
-let editorMsgs = {};
-components.forEach((component) => {
-    let messages = Object.keys(locales).reduce((collection, lang) => {
-        let langMessages = {};
         try {
             let langData = JSON.parse(
                 fs.readFileSync(path.resolve('editor', component, lang + '.json'), 'utf8')
             );
-            Object.keys(langData).forEach((id) => {
+            collection[lang] = langData;
-                langMessages[id] = langData[id].message;
-            });
-            collection[lang] = langMessages;
         } catch (e) {
-            missingLocales.push(lang);
+            missingLocales.push(component + ':' + lang + '\n');
         }
         return collection;
     }, {});
-    
+};
-    let data =
-        '// GENERATED FILE:\n' +
-        'export default ' +
-        JSON.stringify(messages, null, 2) +
-        ';\n';
-    fs.writeFileSync(MSGS_DIR + component + '-msgs.js', data);
-    defaultsDeep(editorMsgs, messages);
-
-    if (missingLocales.length > 0) {
-        process.stdout.write('missing locales: ' + missingLocales.toString());
-        process.exit(1);
-    }
-});
 
 // generate the blocks messages: files are plain key-value JSON
-let blocksMessages = Object.keys(locales).reduce((collection, lang) => {
+let blocksMessages = combineJson('blocks');
-    try {
-        let langData = JSON.parse(
-            fs.readFileSync(path.resolve('editor', 'blocks', lang + '.json'), 'utf8')
-        );
-        collection[lang] = langData;
-    } catch (e) {
-        missingLocales.push(lang);
-    }
-    return collection;
-}, {});
 let blockData =
     '// GENERATED FILE:\n' +
     'export default ' +
@@ -102,6 +72,20 @@ let blockData =
 
 fs.writeFileSync(MSGS_DIR + 'blocks-msgs.js', blockData);
 
+// generate messages for gui components - all files are plain key-value JSON
+let components = ['interface', 'extensions', 'paint-editor'];
+let editorMsgs = {};
+components.forEach((component) => {
+    let messages = combineJson(component);
+    let data =
+        '// GENERATED FILE:\n' +
+        'export default ' +
+        JSON.stringify(messages, null, 2) +
+        ';\n';
+    fs.writeFileSync(MSGS_DIR + component + '-msgs.js', data);
+    defaultsDeep(editorMsgs, messages);
+});
+
 // generate combined editor-msgs file
 let editorData =
     '// GENERATED FILE:\n' +
@@ -109,3 +93,8 @@ let editorData =
     JSON.stringify(editorMsgs, null, 2) +
     ';\n';
 fs.writeFileSync(MSGS_DIR + 'editor-msgs.js', editorData);
+
+if (missingLocales.length > 0) {
+    process.stdout.write('missing locales:\n' + missingLocales.toString());
+    process.exit(1);
+}

scripts/sync_tx_blocks.js

@@ -0,0 +1,72 @@
+#!/usr/bin/env babel-node
+
+/**
+ * @fileoverview
+ * Script to pull blocks translations from transifex and generate the scratch_msgs file.
+ * Blocks uses a flat json file (not Chrome i18n), so needs to be handled separately. Expects
+ * that the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+const args = process.argv.slice(2);
+
+const usage = `
+ Pull blocks translations from Transifex. Usage:
+   node sync_tx_blocks.js path
+     path: where to put the downloaded json files
+   NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+   the Localization page on the GUI wiki for information about setting up Transifex.
+ `;
+// Fail immediately if the TX_TOKEN is not defined
+if (!process.env.TX_TOKEN || args.length < 1) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+
+import fs from 'fs';
+import path from 'path';
+import transifex from 'transifex';
+import async from 'async';
+import locales, {localeMap} from '../src/supported-locales.js';
+
+// Globals
+const PROJECT = 'scratch-editor';
+const RESOURCE = 'blocks';
+const OUTPUT_DIR = path.resolve(args[0]);
+// TODO: convert mode to 'reviewed' before January
+const MODE = {mode: 'default'};
+const CONCURRENCY_LIMIT = 4;
+
+const TX = new transifex({
+    project_slug: PROJECT,
+    credential: 'api:' + process.env.TX_TOKEN
+});
+
+const getLocaleData = (locale, callback) => {
+    let txLocale = localeMap[locale] || locale;
+    TX.translationInstanceMethod(PROJECT, RESOURCE, txLocale, MODE, function (err, data) {
+        if (err) {
+            callback(err);
+        } else {
+            callback(null, {
+                locale: locale,
+                translations: JSON.parse(data)
+            });
+        }
+    });
+};
+
+async.mapLimit(Object.keys(locales), CONCURRENCY_LIMIT, getLocaleData, function (err, values) {
+    if (err) {
+        console.error(err); // eslint-disable-line no-console
+        process.exit(1);
+    }
+    values.forEach(function (translation) {
+        // skip validation as it's handled in scratch-blocks
+        const file = JSON.stringify(translation.translations, null, 4);
+        fs.writeFileSync(
+            `${OUTPUT_DIR}/${translation.locale}.json`,
+            file
+        );
+    });
+});

scripts/sync_tx_src.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+
+/**
+ * @fileoverview
+ * Script to upload a source en.json file to a particular transifex project resource.
+ * Expects that the project and resource have already been defined in Transifex, and that
+ * the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const transifex = require('transifex');
+
+const args = process.argv.slice(2);
+
+const usage = `
+Sync English source strings with Transifex. Usage:
+  node sync_tx_src.js tx-project tx-resource english-json-file
+      tx-project:        the project slug on transifex
+      tx-resource:       the resource slug on transifex
+      english-json-file: path to the en.json source
+  NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+  the Localization page on the GUI wiki for information about setting up Transifex.
+`;
+
+// Exit if missing arguments or TX_TOKEN
+if (args.length < 3 || !process.env.TX_TOKEN) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+
+// Globals
+const PROJECT = args[0];
+const RESOURCE = args[1];
+const TX = new transifex({
+    project_slug: PROJECT,
+    credential: 'api:' + process.env.TX_TOKEN
+});
+
+let en = fs.readFileSync(path.resolve(args[2]));
+en = JSON.parse(en);
+
+// update Transifex with English source
+TX.uploadSourceLanguageMethod(PROJECT, RESOURCE, {content: JSON.stringify(en)}, (err) => {
+    if (err) {
+        console.error(err); // eslint-disable-line no-console
+        process.exit(1);
+    }
+    process.exit(0);
+});

scripts/sync_tx_translations.js

@@ -0,0 +1,76 @@
+#!/usr/bin/env babel-node
+
+/**
+ * @fileoverview
+ * Script to pull translations from transifex and generate the editor-msgs file.
+ * Expects that the project and resource have already been defined in Transifex, and that
+ * the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+const args = process.argv.slice(2);
+
+const usage = `
+ Pull supported language translations from Transifex. Usage:
+   node sync_tx_translations.js tx-project tx-resource path
+     tx-project: project on Transifex (e.g., scratch-editor)
+     tx-resource: resource within the project (e.g., interface)
+     path: where to put the downloaded json files
+   NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+   the Localization page on the GUI wiki for information about setting up Transifex.
+ `;
+// Fail immediately if the TX_TOKEN is not defined
+if (!process.env.TX_TOKEN || args.length < 3) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+
+import fs from 'fs';
+import path from 'path';
+import transifex from 'transifex';
+import async from 'async';
+import {flattenJson, validateTranslations} from './tx_util.js';
+import locales, {localeMap} from '../src/supported-locales.js';
+
+// Globals
+const PROJECT = args[0];
+const RESOURCE = args[1];
+const OUTPUT_DIR = path.resolve(args[2]);
+// TODO: convert mode to 'reviewed' before January
+const MODE = {mode: 'default'};
+const CONCURRENCY_LIMIT = 4;
+
+const TX = new transifex({
+    project_slug: PROJECT,
+    credential: 'api:' + process.env.TX_TOKEN
+});
+
+const getLocaleData = (locale, callback) => {
+    let txLocale = localeMap[locale] || locale;
+    TX.translationInstanceMethod(PROJECT, RESOURCE, txLocale, MODE, function (err, data) {
+        if (err) {
+            callback(err);
+        } else {
+            callback(null, {
+                locale: locale,
+                translations: JSON.parse(data)
+            });
+        }
+    });
+};
+
+async.mapLimit(Object.keys(locales), CONCURRENCY_LIMIT, getLocaleData, function (err, values) {
+    if (err) {
+        console.error(err); // eslint-disable-line no-console
+        process.exit(1);
+    }
+    const source = JSON.parse(flattenJson(values.find(elt => elt.locale === 'en').translations));
+    values.forEach(function (translation) {
+        const file = flattenJson(translation.translations);
+        validateTranslations({locale: translation.locale, translations: JSON.parse(file)}, source);
+        fs.writeFileSync(
+            `${OUTPUT_DIR}/${translation.locale}.json`,
+            file
+        );
+    });
+});

scripts/tx_util.js

@@ -0,0 +1,34 @@
+import assert from 'assert';
+import parse from 'format-message-parse';
+
+const flattenJson = (translations) => {
+    let messages = Object.keys(translations).reduce((collection, id) => {
+        collection[id] = translations[id].message;
+        return collection;
+    }, {});
+    return JSON.stringify(messages, null, 4);
+};
+
+const validMessage = (message, source) => {
+    // this will throw an error if the message is not valid icu
+    const t = parse(message);
+    const s = parse(source);
+    // the syntax tree for both messages should have the same number of elements
+    return t.length === s.length;
+};
+
+const validateTranslations = (translation, source) => {
+    const locale = translation.locale;
+    const translations = translation.translations;
+    const transKeys = Object.keys(translations);
+    const sourceKeys = Object.keys(source);
+    assert.strictEqual(transKeys.length, sourceKeys.length, `locale ${locale} has a different number of message keys`);
+    transKeys.map(item => assert(sourceKeys.includes(item), `locale ${locale} has key ${item} not in the source`));
+    sourceKeys.map(item => assert(transKeys.includes(item), `locale ${locale} is missing key ${item}`));
+    sourceKeys.map(item => assert(
+        validMessage(translations[item], source[item]),
+        `locale ${locale}: "${translations[item]}" is not a valid translation for "${source[item]}"`)
+    );
+};
+
+export {flattenJson, validateTranslations, validMessage};

scripts/validate_translations.js

@@ -0,0 +1,48 @@
+#!/usr/bin/env babel-node
+
+/**
+ * @fileoverview
+ * Script to validate the translation json
+ */
+
+const args = process.argv.slice(2);
+const usage = `
+ Validate translation json. Usage:
+   babel-node validate_translations.js path
+     path: where to find the downloaded json files
+ `;
+// Fail immediately if the TX_TOKEN is not defined
+if (args.length < 1) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+import fs from 'fs';
+import path from 'path';
+import async from 'async';
+import {validateTranslations} from './tx_util.js';
+import locales from '../src/supported-locales.js';
+
+// Globals
+const JSON_DIR = path.resolve(args[0]);
+
+const source = JSON.parse(fs.readFileSync(`${JSON_DIR}/en.json`));
+
+const validate = (locale, callback) => {
+    fs.readFile(`${JSON_DIR}/${locale}.json`, function (err, data) {
+        if (err) callback(err);
+        // let this throw an error if invalid json
+        data = JSON.parse(data);
+        const translations = {
+            locale: locale,
+            translations: data
+        };
+        validateTranslations(translations, source);
+    });
+};
+
+async.each(Object.keys(locales), validate, function (err) {
+    if (err) {
+        console.error(err); // eslint-disable-line no-console
+        process.exit(1);
+    }
+});

src/index.js

@@ -43,7 +43,7 @@ import uk from 'react-intl/locale-data/uk';
 import vi from 'react-intl/locale-data/vi';
 import zh from 'react-intl/locale-data/zh';
 
-import locales, {customLocales, isRtl} from './supported-locales.js';
+import locales, {customLocales, localeMap, isRtl} from './supported-locales.js';
 /*
 locales = {
     'ab': {name: 'Аҧсшәа'},
@@ -127,6 +127,7 @@ for (const lang in customLocales) {
 
 export {
     locales as default,
+    localeMap,
     isRtl,
     localeData // data expected for initializing ReactIntl.addLocaleData
 };

src/supported-locales.js

@@ -79,6 +79,15 @@ const customLocales = {
     }
 };
 
+const localeMap = {
+    'aa-dj': 'aa_DJ',
+    'es-419': 'es_419',
+    // ja-Hira: no map - it's 'ja-Hira' on transifex
+    'pt-br': 'pt_BR',
+    'zh-cn': 'zh_CN',
+    'zh-tw': 'zh_TW'
+};
+
 // list of RTL locales supported, and a function to check whether a locale is RTL
 const rtlLocales = [
     'ar',
@@ -165,4 +174,4 @@ const wwwLocales = {
     'zh-tw': {name: '繁體中文'}
 };
 
-export {locales as default, customLocales, rtlLocales, isRtl, wwwLocales};
+export {locales as default, customLocales, localeMap, rtlLocales, isRtl, wwwLocales};

webpack.config.js

@@ -6,8 +6,18 @@ module.exports = {
     module: {
         rules: [{
             test: /\.js$/,
-            loader: 'babel-loader',
+            include: path.resolve(__dirname, 'src'),
-            include: path.resolve(__dirname, 'src')
+            use: {
+                loader: 'babel-loader',
+                options: {
+                    presets: ['@babel/preset-env'],
+                    plugins: [
+                        '@babel/plugin-proposal-object-rest-spread',
+                        '@babel/plugin-syntax-dynamic-import',
+                        '@babel/plugin-transform-async-to-generator'
+                    ]
+                }
+            }
         }]
     },
     entry: {