Tx import (#1143)

* Transifex transition

Scripts and configuration for transition to Transifex for translation.

* Transifex transition update

changed the name of the place where translations are saved from ‘translations’ to localizations so that we can have a consistent name in both scratchr2, and scratch-www.

A couple of other little clean-ups to make sure that it’s ES5 compliant.
- don’t use const
- don’t use template strings (backticks)

.gitignore

@@ -11,6 +11,8 @@ npm-*
 # Locales
 /locales
 /intl
+/localizations
+
 
 # Elastic Beanstalk Files
 .elasticbeanstalk/*

.tx/config

@@ -0,0 +1,93 @@
+[main]
+host = https://www.transifex.com
+
+[scratch-website.explore-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/explore/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.general-l10njson]
+file_filter = localizations/general/<lang>.json
+source_file = src/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.wedo2-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/wedo2/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.cards-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/cards/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.teacherregistration-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/teacherregistration/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.dmca-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/dmca/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.jobs-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/jobs/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.faq-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/faq/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.about-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/about/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.teacher-faq-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/teachers/faq/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.developers-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/developers/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.things-to-try-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/thingstotry/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.guidelines-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/guidelines/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.educator-landing-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/teachers/landing/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+
+[scratch-website.splash-l10njson]
+file_filter = localizations/${name}/<lang>.json
+source_file = src/views/splash/l10n.json
+source_lang = en
+type = KEYVALUEJSON
+

bin/import-pootle

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+ /*
+    Generate language json files corresponding to l10n files to import from
+    pootle into transifex.
+
+    For example, extract the strings corresponding to splash ids from pootle.
+    For each language, in localizations/splash create
+    fr.json =>
+    {
+        'splash.welcome' : 'Bienvenue',
+        ...
+    }
+    etc.
+ */
+
+var fs = require('fs');
+var path = require('path');
+var routes = require('../src/routes.json');
+var languages = require('../languages.json');
+var localeCompare = require('./lib/locale-compare');
+
+var outputDir = path.resolve(__dirname, '../localizations');
+try {
+    fs.accessSync(outputDir, fs.F_OK);
+} catch (err) {
+    // Doesn't exist - create it.
+    fs.mkdirSync(outputDir);
+}
+
+// general is a special case, do it first
+var l10n = path.resolve(__dirname, '../src/l10n.json');
+localeCompare.writeTranslations('general', l10n, languages);
+
+for (var v in routes) {
+    if (typeof routes[v].redirect !== 'undefined') {
+        continue;
+    }
+    var subdir = routes[v].view.split('/');
+    subdir.pop();
+    l10n = path.resolve(__dirname, '../src/views/' + subdir.join('/') + '/l10n.json');
+    var name = routes[v].name;
+    try {
+        // only import if there is an l10n file
+        fs.accessSync(l10n);
+    } catch (err) {
+        // skip views without l10n files
+        process.stdout.write(`Skipping ${name}, no l10n\n`);
+        continue;
+    }
+    localeCompare.writeTranslations(name, l10n, languages);
+}

bin/init-l10n-src

@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+
+/* Initialize transifex resources from src l10n files
+needs to run in the project root directory, where .tx folder is located
+add parameter 'execute' to run the tx command, otherwise transifex just
+shows what would run
+TBD: replace this with a node transifex module
+*/
+/*
+Don't use template strings (backticks) until scratch-www is switched over
+to ES6. Leaving template string versions as comments.
+ */
+
+// Unfortunately need to execute Synchronously, or tx set gets errors when
+// updating the .tx/config file
+var execSync = require('child_process').execSync;
+var fs = require('fs');
+var path = require('path');
+var routes = require('../src/routes.json');
+
+var cmd = '';
+var execute = '';
+
+// make sure .tx folder exists with config file
+try {
+    //
+    fs.accessSync(path.resolve(process.cwd() + '/.tx/config'));
+} catch (err) {
+    process.stdout.write('Run the script from the directory with .tx folder\n');
+    process.exit(1);
+}
+
+var args = process.argv.slice(2);
+if (args[0] === 'execute') {
+    process.stdout.write('executing tx initializtion\n');
+    execute = '--execute';
+} else {
+    process.stdout.write('Dry run: pass "execute" as a parameter to add --execute switch to commands\n');
+}
+
+
+// set up l10n resources for the scratch-website project as
+// [scratch-website.<view>_l10njson]
+// use 'general' for the root l10n.json
+
+// general is a special case, do that first
+// TODO: ES6
+// cmd = 'tx set --auto-local --source-lang en --type KEYVALUEJSON -r ' +
+//     'scratch-website.general-l10njson \'localizations/general/<lang>.json\' ' +
+//     `--source-file src/l10n.json ${execute}`;
+cmd = 'tx set --auto-local --source-lang en --type KEYVALUEJSON -r ' +
+    'scratch-website.general-l10njson \'localizations/general/<lang>.json\' ' +
+    '--source-file src/l10n.json ' + execute;
+process.stdout.write('Adding general l10n\n');
+execSync(cmd, {stdio:[0,1,2]});
+
+for (var v in routes) {
+    if (typeof routes[v].redirect !== 'undefined') {
+        continue;
+    }
+    var subdir = routes[v].view.split('/');
+    subdir.pop();
+    var l10n = 'src/views/' + subdir.join('/') + '/l10n.json';
+    var name = routes[v].name;
+    try {
+        // only Initialize if there is an l10n file
+        fs.accessSync(l10n);
+        // TODO: ES6
+        // var tx_resource = `scratch-website.${name}-l10njson`;
+        // cmd = `tx set --auto-local --source-lang en --type KEYVALUEJSON -r ${tx_resource}` +
+        // ` \'localizations/${name}/<lang>.json\' --source-file ${l10n} ${execute}`;
+        // process.stdout.write(`Adding ${name} l10n\n`);
+        var tx_resource = 'scratch-website.' + name +'-l10njson';
+        cmd = 'tx set --auto-local --source-lang en --type KEYVALUEJSON -r ' + tx_resource +
+            ' \'localizations/${name}/<lang>.json\' --source-file '+ l10n + ' ' + execute;
+        process.stdout.write('Adding ' + name + ' l10n\n');
+        execSync(cmd, {stdio:'inherit'});
+    } catch (err) {
+        // skip views without l10n files
+        // TODO: ES6
+        // process.stdout.write(`Skipping ${name}, no l10n\n`);
+        process.stdout.write('Skipping ' + name+ ', no l10n\n');
+    }
+}
+
+if (execute === '--execute') {
+    // push all the source files to transifex - force update
+    execSync('tx push -s -f --no-interactive', {stdio:'inherit'});
+}

bin/lib/locale-compare.js

@@ -62,6 +62,22 @@ Helpers.mergeNewTranslations = function (existingTranslations, newTranslations,
     return existingTranslations;
 };
 
+Helpers.mergeNewTranslationsWithoutDefaults = function (existingTranslations, newTranslations, icuTemplate, md5Map) {
+    for (var id in newTranslations) {
+        var md5 = Helpers.getMD5(id);
+        if (md5Map.hasOwnProperty(md5) && newTranslations[id].length > 0) {
+            md5Map[md5].forEach(function (msgId) {
+                existingTranslations[msgId] = newTranslations[id];
+            });
+        }
+    }
+
+    //Fill in empty string for any missing translations
+    for (var icuId in icuTemplate) {
+        if (!existingTranslations.hasOwnProperty(icuId)) existingTranslations[icuId] = '';
+    }
+    return existingTranslations;
+};
 /**
  * Converts a map of icu strings with react-intl id values into a map
  * with md5 hashes of the icu strings as keys and react-intl id values.
@@ -139,6 +155,65 @@ Helpers.getTranslationsForLanguage = function (lang, idsWithICU, md5WithIds, sep
     return translationsByView;
 };
 
+/**
+ * Grabs the translated strings from the po files for the given language and strings
+ * @param  {str}    lang       iso code of the language to use
+ * @param  {object} idsWithICU key: '<viewName>-<react-intl string id>'.
+ *                             value: english strings for translation
+ * @param  {object} md5WithIds key: md5 hash of the english strings for translation.
+ *                             value: '<viewName>-<react-intl string id>'
+ * @return {object}            translations – sub-objects by view containing:
+ *                                          key: '<react-intl string id>'
+ *                                          value: translated version of string
+ *                                                  empty if no translation present
+ */
+Helpers.getTranslationsForLanguageWithoutDefaults = function (lang, idsWithICU, md5WithIds, separator) {
+    var poUiDir = path.resolve(__dirname, '../../node_modules/scratchr2_translations/ui');
+    var jsFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/djangojs.po');
+    var pyFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/django.po');
+
+    var translations = {};
+    separator = separator || ':';
+
+    try {
+        fs.accessSync(jsFile, fs.R_OK);
+        var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
+        translations = Helpers.mergeNewTranslationsWithoutDefaults(translations,
+            jsTranslations, idsWithICU, md5WithIds);
+    } catch (err) {
+        if (err.code !== 'ENOENT') {
+            throw err;
+        }
+    }
+
+    try {
+        fs.accessSync(pyFile, fs.R_OK);
+        var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
+        translations = Helpers.mergeNewTranslationsWithoutDefaults(translations,
+            pyTranslations, idsWithICU, md5WithIds);
+    } catch (err) {
+        if (err.code !== 'ENOENT') {
+            throw err;
+        }
+    }
+
+    var translationsByView = {};
+    for (var id in translations) {
+        var ids = id.split(separator); // [viewName, stringId]
+        var viewName = ids[0];
+        var stringId = ids[1];
+
+        if (!translationsByView.hasOwnProperty(viewName)) {
+            translationsByView[viewName] = {};
+        }
+        if (!translationsByView[viewName].hasOwnProperty(lang)) {
+            translationsByView[viewName][lang] = {};
+        }
+        translationsByView[viewName][lang][stringId] = translations[id];
+    }
+    return translationsByView;
+};
+
 Helpers.writeTranslationsToJS = function (outputDir, viewName, translationObject) {
     var objectString = JSON.stringify(translationObject);
     var fileString = 'window._messages = ' + objectString + ';';
@@ -169,4 +244,30 @@ Helpers.icuToIdMap = function (viewName, ids, separator) {
     return icuToIds;
 };
 
+Helpers.writeTranslations = function (name, l10n, languages) {
+    var ids = require(l10n);
+    var idsWithICU = Helpers.idToICUMap(name, ids);
+    var icuWithIds = Helpers.icuToIdMap(name, ids);
+    var md5WithIds = Helpers.getMD5Map(icuWithIds);
+    var isoCodes = Object.keys(languages);
+    var outputDir = path.resolve(__dirname, '../../localizations/', name);
+    try {
+        fs.accessSync(outputDir, fs.F_OK);
+    } catch (err) {
+        // Doesn't exist - create it.
+        fs.mkdirSync(outputDir);
+    }
+    process.stdout.write(`Writing translations to ${outputDir}\n`);
+
+    for (var isoCode in isoCodes) {
+        if (isoCodes[isoCode] !== 'en'){
+            var translations = Helpers.getTranslationsForLanguageWithoutDefaults(
+                isoCodes[isoCode], idsWithICU, md5WithIds);
+            var fileString = JSON.stringify(translations[name][isoCodes[isoCode]]);
+            fs.writeFileSync(outputDir + '/' + isoCodes[isoCode] + '.json', fileString);
+        }
+    }
+
+};
+
 module.exports = Helpers;