Add scripts for automation

* bumped minor version
* added localeMap for converting between transifex locales and the ones used by Scratch (e.g., pt-br - pt_BR)
* using the transifex node package to automatically sync translations
* added scripts:
  * sync_tx_src: uploads an en.json source file, for use by client packages
  * sync_tx_translations: downloads gui translations, used by this repo, flattens Chrome i18n json into plain key-value json.
  * sync_tx_blocks: same as above, but blocks need slightly different handling
  * validate_translations: check the translation json files for basic translation errors
  * tx_util - methods shared by sync and validate
* simplified build-data because jsons have already been flattened
* added new npm tasks

scripts/build-data.js

@@ -48,52 +48,22 @@ const MSGS_DIR = './locales/';
 mkdirpSync(MSGS_DIR);
 let missingLocales = [];
 
-// generate messages for gui components - files are Chrome i18n format json
-let components = ['interface', 'extensions', 'paint-editor'];
-let editorMsgs = {};
-components.forEach((component) => {
-    let messages = Object.keys(locales).reduce((collection, lang) => {
-        let langMessages = {};
+const combineJson = (component) => {
+    return Object.keys(locales).reduce((collection, lang) => {
         try {
             let langData = JSON.parse(
                 fs.readFileSync(path.resolve('editor', component, lang + '.json'), 'utf8')
             );
-            Object.keys(langData).forEach((id) => {
-                langMessages[id] = langData[id].message;
-            });
-            collection[lang] = langMessages;
+            collection[lang] = langData;
         } catch (e) {
-            missingLocales.push(lang);
+            missingLocales.push(component + ':' + lang + '\n');
         }
         return collection;
     }, {});
-    
-    let data =
-        '// GENERATED FILE:\n' +
-        'export default ' +
-        JSON.stringify(messages, null, 2) +
-        ';\n';
-    fs.writeFileSync(MSGS_DIR + component + '-msgs.js', data);
-    defaultsDeep(editorMsgs, messages);
-
-    if (missingLocales.length > 0) {
-        process.stdout.write('missing locales: ' + missingLocales.toString());
-        process.exit(1);
-    }
-});
+};
 
 // generate the blocks messages: files are plain key-value JSON
-let blocksMessages = Object.keys(locales).reduce((collection, lang) => {
-    try {
-        let langData = JSON.parse(
-            fs.readFileSync(path.resolve('editor', 'blocks', lang + '.json'), 'utf8')
-        );
-        collection[lang] = langData;
-    } catch (e) {
-        missingLocales.push(lang);
-    }
-    return collection;
-}, {});
+let blocksMessages = combineJson('blocks');
 let blockData =
     '// GENERATED FILE:\n' +
     'export default ' +
@@ -102,6 +72,20 @@ let blockData =
 
 fs.writeFileSync(MSGS_DIR + 'blocks-msgs.js', blockData);
 
+// generate messages for gui components - all files are plain key-value JSON
+let components = ['interface', 'extensions', 'paint-editor'];
+let editorMsgs = {};
+components.forEach((component) => {
+    let messages = combineJson(component);
+    let data =
+        '// GENERATED FILE:\n' +
+        'export default ' +
+        JSON.stringify(messages, null, 2) +
+        ';\n';
+    fs.writeFileSync(MSGS_DIR + component + '-msgs.js', data);
+    defaultsDeep(editorMsgs, messages);
+});
+
 // generate combined editor-msgs file
 let editorData =
     '// GENERATED FILE:\n' +
@@ -109,3 +93,8 @@ let editorData =
     JSON.stringify(editorMsgs, null, 2) +
     ';\n';
 fs.writeFileSync(MSGS_DIR + 'editor-msgs.js', editorData);
+
+if (missingLocales.length > 0) {
+    process.stdout.write('missing locales:\n' + missingLocales.toString());
+    process.exit(1);
+}

scripts/sync_tx_blocks.js

@@ -0,0 +1,72 @@
+#!/usr/bin/env babel-node
+
+/**
+ * @fileoverview
+ * Script to pull blocks translations from transifex and generate the scratch_msgs file.
+ * Blocks uses a flat json file (not Chrome i18n), so needs to be handled separately. Expects
+ * that the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+const args = process.argv.slice(2);
+
+const usage = `
+ Pull blocks translations from Transifex. Usage:
+   node sync_tx_blocks.js path
+     path: where to put the downloaded json files
+   NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+   the Localization page on the GUI wiki for information about setting up Transifex.
+ `;
+// Fail immediately if the TX_TOKEN is not defined
+if (!process.env.TX_TOKEN || args.length < 1) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+
+import fs from 'fs';
+import path from 'path';
+import transifex from 'transifex';
+import async from 'async';
+import locales, {localeMap} from '../src/supported-locales.js';
+
+// Globals
+const PROJECT = 'scratch-editor';
+const RESOURCE = 'blocks';
+const OUTPUT_DIR = path.resolve(args[0]);
+// TODO: convert mode to 'reviewed' before January
+const MODE = {mode: 'default'};
+const CONCURRENCY_LIMIT = 4;
+
+const TX = new transifex({
+    project_slug: PROJECT,
+    credential: 'api:' + process.env.TX_TOKEN
+});
+
+const getLocaleData = (locale, callback) => {
+    let txLocale = localeMap[locale] || locale;
+    TX.translationInstanceMethod(PROJECT, RESOURCE, txLocale, MODE, function (err, data) {
+        if (err) {
+            callback(err);
+        } else {
+            callback(null, {
+                locale: locale,
+                translations: JSON.parse(data)
+            });
+        }
+    });
+};
+
+async.mapLimit(Object.keys(locales), CONCURRENCY_LIMIT, getLocaleData, function (err, values) {
+    if (err) {
+        console.error(err); // eslint-disable-line no-console
+        process.exit(1);
+    }
+    values.forEach(function (translation) {
+        // skip validation as it's handled in scratch-blocks
+        const file = JSON.stringify(translation.translations, null, 4);
+        fs.writeFileSync(
+            `${OUTPUT_DIR}/${translation.locale}.json`,
+            file
+        );
+    });
+});

scripts/sync_tx_src.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+
+/**
+ * @fileoverview
+ * Script to upload a source en.json file to a particular transifex project resource.
+ * Expects that the project and resource have already been defined in Transifex, and that
+ * the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const transifex = require('transifex');
+
+const args = process.argv.slice(2);
+
+const usage = `
+Sync English source strings with Transifex. Usage:
+  node sync_tx_src.js tx-project tx-resource english-json-file
+      tx-project:        the project slug on transifex
+      tx-resource:       the resource slug on transifex
+      english-json-file: path to the en.json source
+  NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+  the Localization page on the GUI wiki for information about setting up Transifex.
+`;
+
+// Exit if missing arguments or TX_TOKEN
+if (args.length < 3 || !process.env.TX_TOKEN) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+
+// Globals
+const PROJECT = args[0];
+const RESOURCE = args[1];
+const TX = new transifex({
+    project_slug: PROJECT,
+    credential: 'api:' + process.env.TX_TOKEN
+});
+
+let en = fs.readFileSync(path.resolve(args[2]));
+en = JSON.parse(en);
+
+// update Transifex with English source
+TX.uploadSourceLanguageMethod(PROJECT, RESOURCE, {content: JSON.stringify(en)}, (err) => {
+    if (err) {
+        console.error(err); // eslint-disable-line no-console
+        process.exit(1);
+    }
+    process.exit(0);
+});

scripts/sync_tx_translations.js

@@ -0,0 +1,76 @@
+#!/usr/bin/env babel-node
+
+/**
+ * @fileoverview
+ * Script to pull translations from transifex and generate the editor-msgs file.
+ * Expects that the project and resource have already been defined in Transifex, and that
+ * the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+const args = process.argv.slice(2);
+
+const usage = `
+ Pull supported language translations from Transifex. Usage:
+   node sync_tx_translations.js tx-project tx-resource path
+     tx-project: project on Transifex (e.g., scratch-editor)
+     tx-resource: resource within the project (e.g., interface)
+     path: where to put the downloaded json files
+   NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+   the Localization page on the GUI wiki for information about setting up Transifex.
+ `;
+// Fail immediately if the TX_TOKEN is not defined
+if (!process.env.TX_TOKEN || args.length < 3) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+
+import fs from 'fs';
+import path from 'path';
+import transifex from 'transifex';
+import async from 'async';
+import {flattenJson, validateTranslations} from './tx_util.js';
+import locales, {localeMap} from '../src/supported-locales.js';
+
+// Globals
+const PROJECT = args[0];
+const RESOURCE = args[1];
+const OUTPUT_DIR = path.resolve(args[2]);
+// TODO: convert mode to 'reviewed' before January
+const MODE = {mode: 'default'};
+const CONCURRENCY_LIMIT = 4;
+
+const TX = new transifex({
+    project_slug: PROJECT,
+    credential: 'api:' + process.env.TX_TOKEN
+});
+
+const getLocaleData = (locale, callback) => {
+    let txLocale = localeMap[locale] || locale;
+    TX.translationInstanceMethod(PROJECT, RESOURCE, txLocale, MODE, function (err, data) {
+        if (err) {
+            callback(err);
+        } else {
+            callback(null, {
+                locale: locale,
+                translations: JSON.parse(data)
+            });
+        }
+    });
+};
+
+async.mapLimit(Object.keys(locales), CONCURRENCY_LIMIT, getLocaleData, function (err, values) {
+    if (err) {
+        console.error(err); // eslint-disable-line no-console
+        process.exit(1);
+    }
+    const source = JSON.parse(flattenJson(values.find(elt => elt.locale === 'en').translations));
+    values.forEach(function (translation) {
+        const file = flattenJson(translation.translations);
+        validateTranslations({locale: translation.locale, translations: JSON.parse(file)}, source);
+        fs.writeFileSync(
+            `${OUTPUT_DIR}/${translation.locale}.json`,
+            file
+        );
+    });
+});

scripts/tx_util.js

@@ -0,0 +1,34 @@
+import assert from 'assert';
+import parse from 'format-message-parse';
+
+const flattenJson = (translations) => {
+    let messages = Object.keys(translations).reduce((collection, id) => {
+        collection[id] = translations[id].message;
+        return collection;
+    }, {});
+    return JSON.stringify(messages, null, 4);
+};
+
+const validMessage = (message, source) => {
+    // this will throw an error if the message is not valid icu
+    const t = parse(message);
+    const s = parse(source);
+    // the syntax tree for both messages should have the same number of elements
+    return t.length === s.length;
+};
+
+const validateTranslations = (translation, source) => {
+    const locale = translation.locale;
+    const translations = translation.translations;
+    const transKeys = Object.keys(translations);
+    const sourceKeys = Object.keys(source);
+    assert.strictEqual(transKeys.length, sourceKeys.length, `locale ${locale} has a different number of message keys`);
+    transKeys.map(item => assert(sourceKeys.includes(item), `locale ${locale} has key ${item} not in the source`));
+    sourceKeys.map(item => assert(transKeys.includes(item), `locale ${locale} is missing key ${item}`));
+    sourceKeys.map(item => assert(
+        validMessage(translations[item], source[item]),
+        `locale ${locale}: "${translations[item]}" is not a valid translation for "${source[item]}"`)
+    );
+};
+
+export {flattenJson, validateTranslations, validMessage};

scripts/validate_translations.js

@@ -0,0 +1,48 @@
+#!/usr/bin/env babel-node
+
+/**
+ * @fileoverview
+ * Script to validate the translation json
+ */
+
+const args = process.argv.slice(2);
+const usage = `
+ Validate translation json. Usage:
+   babel-node validate_translations.js path
+     path: where to find the downloaded json files
+ `;
+// Fail immediately if the TX_TOKEN is not defined
+if (args.length < 1) {
+    process.stdout.write(usage);
+    process.exit(1);
+}
+import fs from 'fs';
+import path from 'path';
+import async from 'async';
+import {validateTranslations} from './tx_util.js';
+import locales from '../src/supported-locales.js';
+
+// Globals
+const JSON_DIR = path.resolve(args[0]);
+
+const source = JSON.parse(fs.readFileSync(`${JSON_DIR}/en.json`));
+
+const validate = (locale, callback) => {
+    fs.readFile(`${JSON_DIR}/${locale}.json`, function (err, data) {
+        if (err) callback(err);
+        // let this throw an error if invalid json
+        data = JSON.parse(data);
+        const translations = {
+            locale: locale,
+            translations: data
+        };
+        validateTranslations(translations, source);
+    });
+};
+
+async.each(Object.keys(locales), validate, function (err) {
+    if (err) {
+        console.error(err); // eslint-disable-line no-console
+        process.exit(1);
+    }
+});