Automate translation update (#1752)

* Add transifex node package to support syncing scripts
* Add scripts to push source `en.json` to Transifex and download translations to generate `scratch_msgs.js`
* Adds a test to ensure that changes to `msg/messages.js` have been reflected in the source and generated `scratch_msgs.js`
* gets the list of supported languages from scratch-l10n

i18n/sync_tx_src.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+
+/**
+ * @fileoverview
+ * Script to upload a source en.json file to a particular transifex project resource.
+ * Expects that the project and resource have already been defined in Transifex, and that
+ * the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const transifex = require('transifex');
+
+const args = process.argv.slice(2);
+
+const usage = `
+Sync English source strings with Transifex. Usage:
+  node sync_tx_src.js tx-project tx-resource english-json-file
+      tx-project:        the project slug on transifex
+      tx-resource:       the resource slug on transifex
+      english-json-file: path to the en.json source
+  NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+  the Localization page on the GUI wiki for information about setting up Transifex.
+`;
+
+// Exit if missing arguments or TX_TOKEN
+if (args.length < 3 || !process.env.TX_TOKEN) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+
+// Globals
+const PROJECT = args[0];
+const RESOURCE = args[1];
+const TX = new transifex({
+  project_slug: PROJECT,
+  credential: 'api:' + process.env.TX_TOKEN
+});
+
+let en = fs.readFileSync(path.resolve(args[2]));
+en = JSON.parse(en);
+
+// update Transifex with English source
+TX.uploadSourceLanguageMethod(PROJECT, RESOURCE, {content: JSON.stringify(en)}, (err, status) => {
+  if (err) {
+    console.error(err);
+    process.exit(1);
+  }
+  process.exit(0);
+});

i18n/sync_tx_translations.js

@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+
+/**
+ * @fileoverview
+ * Script to pull translations for blocks from transifex and generate the scratch_msgs file.
+ * Expects that the project and resource have already been defined in Transifex, and that
+ * the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+ const usage = `
+ Pull supported language translations from Transifex. Usage:
+   node sync_tx_translations.js
+   NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+   the Localization page on the GUI wiki for information about setting up Transifex.
+ `;
+// Fail immediately if the TX_TOKEN is not defined
+if (!process.env.TX_TOKEN || process.argv.length !== 2) {
+  process.stdout.write(usage);
+  process.exit(1);
+};
+
+const fs = require('fs');
+const path = require('path');
+const assert = require('assert');
+const transifex = require('transifex');
+const locales = require('scratch-l10n').default;
+
+// Globals
+const PATH_OUTPUT = path.resolve(__dirname, '../msg');
+const PROJECT = 'scratch-editor'
+const RESOURCE = 'blocks';
+// TODO: convert mode to 'reviewed' before January
+const MODE = {mode: 'default'};
+
+const TX = new transifex({
+  project_slug: PROJECT,
+  credential: 'api:' + process.env.TX_TOKEN
+});
+
+let en = fs.readFileSync(path.resolve(__dirname, '../msg/json/en.json'));
+en = JSON.parse(en);
+const enKeys = Object.keys(en).sort().toString();
+
+// Check that translation is valid:
+// entry: array [key, translation]  corresponding to a single string from <locale>.json
+// - messages with placeholders have the same number of placeholders
+// - messages must not have newlines embedded
+const validateEntry = function (entry) {
+    const re = /(%\d)/g;
+    const [key, translation] = entry;
+    const enMatch = en[key].match(re);
+    const tMatch = translation.match(re);
+    const enCount = enMatch ? enMatch.length : 0;
+    const tCount = tMatch ? tMatch.length : 0;
+    assert.strictEqual(tCount, enCount, `${key}:${en[key]} - "${translation}" placeholder mismatch`);
+    if (enCount > 0) {
+
+      assert.notStrictEqual(tMatch, null, `${key} is missing a placeholder: ${translation}`);
+      assert.strictEqual(
+          tMatch.sort().toString(),
+          enMatch.sort().toString(),
+          `${key} is missing or has duplicate placeholders: ${translation}`
+      );
+    }
+    assert.strictEqual(translation.match(/[\n]/), null, `${key} contains a newline character ${translation}`);
+};
+
+const validate = function (json, name) {
+    assert.strictEqual(Object.keys(json).sort().toString(), enKeys, `${name}: Locale json keys do not match en.json`);
+    Object.entries(json).forEach(validateEntry);
+};
+
+let file = `// This file was automatically generated.  Do not modify.
+
+'use strict';
+
+goog.provide('Blockly.ScratchMsgs.allLocales');
+
+goog.require('Blockly.ScratchMsgs');
+
+`;
+
+let localeMap = {
+  'aa-dj': 'aa_DJ',
+  'es-419': 'es_419',
+  'pt-br': 'pt_BR',
+  'zh-cn': 'zh_CN',
+  'zh-tw': 'zh_TW'
+};
+
+function getLocaleData (locale) {
+  let txLocale = localeMap[locale] || locale;
+  return new Promise (function (resolve, reject) {
+    TX.translationInstanceMethod(PROJECT, RESOURCE, txLocale, MODE, function (err, data) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve({
+          locale: locale,
+          translations: JSON.parse(data)
+        });
+      }
+    })
+  })
+};
+
+Promise.all(Object.keys(locales).map(getLocaleData)).then(function (values) {
+  values.forEach(function (translation) {
+    validate(translation.translations, translation.locale);
+    file += '\n';
+    file += `Blockly.ScratchMsgs.locales["${translation.locale}"] =\n`;
+    file += JSON.stringify(translation.translations, null, 4);
+    file += ';\n';
+  });
+  file += '// End of combined translations\n';
+  // write combined file
+  fs.writeFileSync(`${PATH_OUTPUT}/scratch_msgs.js`, file);
+}).catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

i18n/test_scratch_msgs.js

@@ -0,0 +1,87 @@
+const es = require('event-stream');
+const fs = require('fs');
+const path = require('path');
+const assert = require('assert');
+
+// current locale and keys for the locale
+let locale = '';
+let keys = [];
+
+// current English keys
+let en = fs.readFileSync(path.resolve(__dirname, '../msg/json/en.json'));
+en = JSON.parse(en);
+const enKeys = Object.keys(en);
+
+// File paths
+const PATH_INPUT = path.resolve(__dirname, '../msg/scratch_msgs.js');
+
+// Match lines of the scratch_msgs file
+// Blockly.ScratchMsgs.locales indicates the start of a new locale
+// ": " marks a "key": "value" pair
+// Also match the end of the generated file so the last set of keys can be checked
+const match = function (str) {
+    if (str.indexOf('Blockly.ScratchMsgs.locales') !== 0) return true;
+    if (str.indexOf('": "') !== 0) return true;
+    if (str.indexOf('End of combined translations') !== 0) return true;
+    return false;
+}
+
+// Extract key and value from message definition, or locale when it changes
+const extract = function (str) {
+    let m = str.match(/locales\["(.+)"\]/);
+    if (m) {
+      // locale changed
+      return m[1];
+    }
+    m = str.match(/"(.*)": "(.*)",?$/);
+    if (m) {
+      return {
+        key: m[1],
+        value: m[2]
+      }
+    }
+    // return a string for the end of the file so that validate will check the last set of keys
+    m = str.match(/^\/\/ End of combined translations$/);
+    if (m) return 'last';
+    return null;
+};
+
+const validateKeys = function () {
+    // ignore empty keys first time through
+    if (keys.length === 0) return;
+    assert.strictEqual(keys.length, Object.keys(en).length, `scratch_msgs-${locale}: number of keys doesn't match`);
+    keys.map(item => assert(enKeys.includes(item), `scratch_msgs-${locale}: has key ${item} not in en`));
+    enKeys.map(item => assert(keys.includes(item), `scratch_msgs-${locale}: is missing key ${item}`));
+}
+
+// Stream input and push each match to the storage object
+const stream = fs.createReadStream(PATH_INPUT);
+stream
+    .pipe(es.split('\n'))
+    .pipe(es.mapSync(function (str) {
+        if (!match(str)) return;
+        const result = extract(str);
+        if (!result) return;
+        if (typeof result === 'string') {
+          // locale changed or end of file, validate the current collection of keys
+          try {
+              validateKeys();
+          }
+          catch (err) {
+            console.error('Key validation FAILED: %O', err);
+            process.exit(1);
+          }
+          // change locale, and reset keys array
+          locale = result;
+          keys = [];
+        } else {
+          keys.push(result.key);
+        }
+    }))
+    .pipe(es.wait(function (err) {
+        if (err) {
+          console.err(err);
+          process.exit(1);
+        }
+        process.exit(0)
+    }));

package.json

@@ -17,11 +17,13 @@
     "test:setup": "tests/scripts/test_setup.sh",
     "test:unit": "node tests/jsunit/test_runner.js",
     "test:lint": "eslint .",
-    "test:messages": "node i18n/js_to_json.js",
+    "test:messages": "npm run translate && node i18n/test_scratch_msgs.js",
     "test": "npm run test:lint && npm run test:messages && npm run test:setup && npm run test:unit",
     "version": "json -f package.json -I -e \"this.repository.sha = '$(git log -n1 --pretty=format:%H)'\"",
-    "translate": "npm run test:messages && node i18n/json_to_js.js",
-    "translate:update": "node i18n/create_scratch_msgs.js"
+    "translate": "node i18n/js_to_json.js && node i18n/json_to_js.js",
+    "translate:sync:src": "node i18n/sync_tx_src.js scratch-editor blocks msg/json/en.json",
+    "translate:sync:translations": "node i18n/sync_tx_translations.js",
+    "translate:update": "npm run translate:sync:src && npm run translate:sync:translations"
   },
   "dependencies": {
     "exports-loader": "0.6.3",
@@ -39,6 +41,8 @@
     "graceful-fs": "4.1.11",
     "json": "9.0.4",
     "rimraf": "2.6.2",
+    "scratch-l10n": "3.0.20181024221158",
+    "transifex": "1.5.0",
     "travis-after-all": "1.4.4",
     "uglifyjs-webpack-plugin": "^1.2.5",
     "webdriverio": "4.8.0",