scratchfoundation/scratch-www
1
0
Fork 0
mirror of https://github.com/scratchfoundation/scratch-www.git synced 2025-03-14 15:09:59 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

New scripts to generate translations from Transifex

Browse source 
Added tx-import, expects TX_TOKEN environment variable to be set to an API token for Transifex. If no API token exists it’ll just use the default English strings for everything.

Removed the localization spot checks because they were checking tha json got built from po files. With Tx-import the files are already json, and may possibly be missing.
This commit is contained in:
chrisgarrity 2017-02-13 08:29:10 -05:00
parent 9255185716
commit 82785435bf
24 changed files with 264 additions and 813 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
.tx/config
View file

@ -1,8 +1,9 @@
[main]
host = https://www.transifex.com
lang_map = zh_CN:zh-cn, zh_TW:zh-tw, pt_BR:pt-br
[scratch-website.explore-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/explore/<lang>.json
source_file = src/views/explore/l10n.json
source_lang = en
type = KEYVALUEJSON
@ -14,80 +15,86 @@ source_lang = en
type = KEYVALUEJSON
[scratch-website.wedo2-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/wedo2/<lang>.json
source_file = src/views/wedo2/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.cards-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/cards/<lang>.json
source_file = src/views/cards/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.teacherregistration-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/teacherregistration/<lang>.json
source_file = src/views/teacherregistration/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.dmca-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/dmca/<lang>.json
source_file = src/views/dmca/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.jobs-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/jobs/<lang>.json
source_file = src/views/jobs/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.faq-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/faq/<lang>.json
source_file = src/views/faq/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.about-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/about/<lang>.json
source_file = src/views/about/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.teacher-faq-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/teacher-faq/<lang>.json
source_file = src/views/teachers/faq/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.developers-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/developers/<lang>.json
source_file = src/views/developers/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.things-to-try-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/things-to-try/<lang>.json
source_file = src/views/thingstotry/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.guidelines-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/guidelines/<lang>.json
source_file = src/views/guidelines/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.educator-landing-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/educator-landing/<lang>.json
source_file = src/views/teachers/landing/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.splash-l10njson]
file_filter = localizations/${name}/<lang>.json
file_filter = localizations/splash/<lang>.json
source_file = src/views/splash/l10n.json
source_lang = en
type = KEYVALUEJSON
[scratch-website.microworlds-homepage-l10njson]
file_filter = localizations/microworlds-homepage/<lang>.json
source_file = src/views/microworldshomepage/l10n.json
source_lang = en
type = KEYVALUEJSON

3
Makefile
View file

@ -24,7 +24,8 @@ deploy:
@make sync
translations:
./bin/build-locales intl
./bin/tx-import localizations
./bin/build-locales localizations intl
webpack:
$(WEBPACK) --bail

170
bin/build-locales
View file

@ -1,18 +1,15 @@
#!/usr/bin/env node
/*
Converts the existing .po translation files in the module to JSON files.
Requires po2json in order to work. Takes as input a directory
in which to store the resulting json translation files.
Generates javascript translation files for each view from the individual language
json files downloaded from Transifex for each view. Each view also includes the
general translations.
Takes in as an argument an output directory to put translation files.
Searches for files named `l10n.json` in the `src/views/` directory to get
template english strings (as well as the general template at `src/l10n.json`).
Expects two inputs, the directory for the input json files, and an output
to put the generated js files into.
It compiles the template strings into a flat object that is compared against the
translations in the .po files from the `scratchr2_translations` dependency, using
an md5 of the template string without whitespace, and an md5 of the .po msgid string
without whitespace.
The input directory is to have subfolders for each view, with json files named by
langage code. E.g., localizations/about/fr.json
The output files are javascript files that declare objects by locale. Each locale
has a sub-object with FormattedMessage ids as keys, and translated strings as
@ -36,26 +33,54 @@
'''
*/
var fs = require('fs');
var merge = require('lodash.merge');
var path = require('path');
var merge = require('lodash.merge');
var async = require('async');
var languages = require('../languages.json');
var localeCompare = require('./lib/locale-compare');
var localizedUrls = require('./lib/localized-urls');
var routes = require('../src/routes.json');
// -----------------------------------------------------------------------------
// Utility function
// -----------------------------------------------------------------------------
var writeIntlFile = function (outputDir, view, translations, callback) {
var fileString = 'window._messages = ' + JSON.stringify(translations) + ';';
var fileName = outputDir + '/' + view + '.intl.js';
fs.writeFile(fileName, fileString, 'utf8', function (err) {
if (err) {
callback(err);
} else {
callback();
}
});
};
// -----------------------------------------------------------------------------
// Main script
// -----------------------------------------------------------------------------
var args = process.argv.slice(2);
if (!args.length) {
process.stdout.write('A destination directory must be specified.');
process.stdout.write('A localizations directory must be specified.\n');
process.exit(1);
}
var localesDir = path.resolve(__dirname, '../', args.shift());
try {
fs.accessSync(localesDir, fs.F_OK);
} catch (err) {
// Doesn't exist - create it.
process.stdout.write('Fatal error: No localizations directory.\n');
process.exit(1);
}
if (!args.length) {
process.stdout.write('A destination directory must be specified.\n');
process.exit(1);
}
var outputDir = path.resolve(__dirname, '../', args[0]);
try {
fs.accessSync(outputDir, fs.F_OK);
@ -66,22 +91,32 @@ try {
// get global locale strings first.
var globalTemplateFile = path.resolve(__dirname, '../src/l10n.json');
var ids = require(globalTemplateFile);
// var ids = JSON.parse(fs.readFileSync(globalTemplateFile, 'utf8'));
// message key with english string values (i.e. default values)
var viewLocales = {
general: {
en: ids
// generate object with all translations of general strings;
// every view includes these strings
var generalLocales = {'en': JSON.parse(fs.readFileSync(globalTemplateFile, 'utf8'))};
for ( var l in languages ) {
try {
var langTranslations = path.resolve(__dirname, '../', localesDir, 'general', l + '.json');
fs.accessSync(langTranslations);
generalLocales[l] = JSON.parse(fs.readFileSync(langTranslations, 'utf8'));
} catch (err) {
// just use english
generalLocales[l] = generalLocales['en'];
}
};
var idsWithICU = localeCompare.idToICUMap('general', ids);
var icuWithIds = localeCompare.icuToIdMap('general', ids);
}
// default strings (en), for all views
var defaultLocales = {
//e.g.
// 'explore: {'explore.recent': 'recent'}
// 'about': {'about.title': 'about Scratch'}
// ... etc
};
var views = [];
var localizedAssetUrls = {};
// start with all views, and remove localized ones as they are iterated over
// start with english default for all views
for (var v in routes) {
if (typeof routes[v].redirect !== 'undefined') {
continue;
@ -92,15 +127,10 @@ for (var v in routes) {
var subdir = routes[v].view.split('/');
subdir.pop();
var l10n = path.resolve(__dirname, '../src/views/' + subdir.join('/') + '/l10n.json');
ids = require(l10n);
viewLocales[routes[v].name] = {
en: ids
};
idsWithICU = merge(idsWithICU, localeCompare.idToICUMap(routes[v].name, ids));
// Note: if lodash.merge gets updated to 4.0 or higher this needs to be mergeWith instead
icuWithIds = merge(icuWithIds, localeCompare.icuToIdMap(routes[v].name, ids), localeCompare.customMerge);
var viewIds = JSON.parse(fs.readFileSync(l10n, 'utf8'));
defaultLocales[routes[v].name] = viewIds;
} catch (err) {
if (err.code !== 'MODULE_NOT_FOUND') {
if (err.code !== 'ENOENT') {
throw err;
}
@ -120,6 +150,7 @@ for (var v in routes) {
localizedAssetUrls[routes[v].name] = {};
var assetUrls = require(l10nStatic);
localizedAssetUrls[routes[v].name]['en'] = assetUrls;
for (var lang in languages) {
localizedAssetUrls[routes[v].name][lang] = {};
var langUrls = localizedUrls[lang] || {};
@ -145,26 +176,61 @@ for (var v in routes) {
}
}
// md5 of english strings with message key as the value for searching po files.
// Sample structure: { 'sdfas43534sdfasdf': 'general-general.blah', 'lkjfasdf4t342asdfa': 'about-about.blah' }
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var allLangs = Object.keys(languages);
// Get ui localization strings first
var isoCodes = Object.keys(languages);
for (var isoCode in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[isoCode], idsWithICU, md5WithIds);
for (var messageId in translations) {
viewLocales[messageId] = merge(viewLocales[messageId], translations[messageId]);
}
}
async.forEachLimit(views, 5, function (view, cb) {
//start with localizedAssets, merge the view translations
var viewLocales = {};
viewLocales['en'] = merge({}, generalLocales['en'], defaultLocales[view]);
for (var view in views) {
var viewTranslations = viewLocales['general'];
if (views[view] in viewLocales) {
viewTranslations = merge(viewLocales[views[view]], viewTranslations);
if ( defaultLocales.hasOwnProperty(view) ) {
// merge view specific english strings, first then other languages
process.stdout.write('Merging translations for ' + view + '\n');
async.forEach(allLangs, function (isoCode, cb) {
var translationsFile = path.resolve(__dirname, '../', localesDir, view, isoCode + '.json');
fs.readFile(translationsFile, 'utf8', function (err, data) {
if (err) {
if (err.code === 'ENOENT') {
if (isoCode !== 'en') {
process.stdout.write('No translations for ' + view + isoCode + ', using english\n');
viewLocales[isoCode] = merge({}, generalLocales[isoCode], defaultLocales[view]);
}
return cb();
} else {
return cb(err);
}
}
try {
viewLocales[isoCode] = merge({}, generalLocales[isoCode], JSON.parse(data));
} catch (e) {
return cb(e);
}
cb();
});
}, function (err) {
if (err) process.stdout.write('Error merging translations for view: ' + view + '\n' + err + '\n');
var viewTranslations = merge({}, viewLocales, localizedAssetUrls[view]);
writeIntlFile(outputDir, view, viewTranslations, function (err) {
if (err) {
process.stdout.write('Failed to save: ' + view + '\n');
return cb(err);
}
});
});
} else {
var viewTranslations = merge({}, viewLocales, localizedAssetUrls[view]);
writeIntlFile(outputDir, view, viewTranslations, function (err) {
if (err) {
process.stdout.write('Failed to save: ' + view + '\n');
return cb(err);
}
});
}
if (views[view] in localizedAssetUrls) {
viewTranslations = merge(viewTranslations, localizedAssetUrls[[views[view]]]);
cb();
}, function (err) {
if (err) {
process.stdout.write('Writing intl files completed with errors\n');
}
localeCompare.writeTranslationsToJS(outputDir, views[view], viewTranslations);
}
return;
});

2
bin/init-l10n-src
View file

@ -72,7 +72,7 @@ for (var v in routes) {
// process.stdout.write(`Adding ${name} l10n\n`);
var tx_resource = 'scratch-website.' + name +'-l10njson';
cmd = 'tx set --auto-local --source-lang en --type KEYVALUEJSON -r ' + tx_resource +
' \'localizations/${name}/<lang>.json\' --source-file '+ l10n + ' ' + execute;
' \'localizations/' + name + '/<lang>.json\' --source-file '+ l10n + ' ' + execute;
process.stdout.write('Adding ' + name + ' l10n\n');
execSync(cmd, {stdio:'inherit'});
} catch (err) {

273
bin/lib/locale-compare.js
View file

@ -1,273 +0,0 @@
// -----------------------------------------------------------------------------
// Helper Methods for build-locales node script.
// -----------------------------------------------------------------------------
var crypto = require('crypto');
var fs = require('fs');
var path = require('path');
var po2icu = require('po2icu');
var isArray = require('lodash.isarray');
var Helpers = {};
/**
* Get the md5 has of a string with whitespace removed.
*
* @param {string} string a string
* @return {string} an md5 hash of the string in hex.
*/
Helpers.getMD5 = function (string) {
var cleanedString = string.replace(/\s+/g, '');
return crypto.createHash('md5').update(cleanedString, 'utf8').digest('hex');
};
/**
* Customizer for icuWithIds merge.
* If icu key already has an id value, concatenate additional ids instead
* of replacing.
*
* @param {array} objVal current value
* @param {string} srcVal new value
* @return {array} objVal with srcVal concatenated
*/
Helpers.customMerge = function (objVal, srcVal) {
if (isArray(objVal)) {
return objVal.concat(srcVal);
}
};
/*
Existing translations should be in the key value format specified by react-intl (i.e.
formatted message id, with icu string as the value). New Translations should be in the
format returned by po2icu (i.e. a source language icu string for key, and a localized
language icu string for value).
ICU Map is an object in the reverse react-intl formatting (icu string as key), which will
help determine if the translation belongs in www currently.
*/
Helpers.mergeNewTranslations = function (existingTranslations, newTranslations, icuTemplate, md5Map) {
for (var id in newTranslations) {
var md5 = Helpers.getMD5(id);
if (md5Map.hasOwnProperty(md5) && newTranslations[id].length > 0) {
md5Map[md5].forEach(function (msgId) {
existingTranslations[msgId] = newTranslations[id];
});
}
}
//Fill in defaults
for (var icuId in icuTemplate) {
if (!existingTranslations.hasOwnProperty(icuId)) existingTranslations[icuId] = icuTemplate[icuId];
}
return existingTranslations;
};
Helpers.mergeNewTranslationsWithoutDefaults = function (existingTranslations, newTranslations, icuTemplate, md5Map) {
for (var id in newTranslations) {
var md5 = Helpers.getMD5(id);
if (md5Map.hasOwnProperty(md5) && newTranslations[id].length > 0) {
md5Map[md5].forEach(function (msgId) {
existingTranslations[msgId] = newTranslations[id];
});
}
}
//Fill in empty string for any missing translations
for (var icuId in icuTemplate) {
if (!existingTranslations.hasOwnProperty(icuId)) existingTranslations[icuId] = '';
}
return existingTranslations;
};
/**
* Converts a map of icu strings with react-intl id values into a map
* with md5 hashes of the icu strings as keys and react-intl id values.
* This is done so as to eliminate potential po conversion misses that
* could be caused by different white space formatting between po and icu.
*
* The md5 is generated after all white space is removed from the string.
*
* @param {object} idICUMap map where key=icuString, value=react-intl id
*
* @return {object}
*/
Helpers.getMD5Map = function (ICUIdMap) {
var md5Map = {};
for (var icu in ICUIdMap) {
var md5 = Helpers.getMD5(icu);
md5Map[md5] = ICUIdMap[icu];
}
return md5Map;
};
/**
* Grabs the translated strings from the po files for the given language and strings
* @param {str} lang iso code of the language to use
* @param {object} idsWithICU key: '<viewName>-<react-intl string id>'.
* value: english strings for translation
* @param {object} md5WithIds key: md5 hash of the english strings for translation.
* value: '<viewName>-<react-intl string id>'
* @return {object} translations – sub-objects by view containing:
* key: '<react-intl string id>'
* value: translated version of string
*/
Helpers.getTranslationsForLanguage = function (lang, idsWithICU, md5WithIds, separator) {
var poUiDir = path.resolve(__dirname, '../../node_modules/scratchr2_translations/ui');
var jsFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/djangojs.po');
var pyFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/django.po');
var translations = {};
separator = separator || ':';
try {
fs.accessSync(jsFile, fs.R_OK);
var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
translations = Helpers.mergeNewTranslations(translations, jsTranslations, idsWithICU, md5WithIds);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
try {
fs.accessSync(pyFile, fs.R_OK);
var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
translations = Helpers.mergeNewTranslations(translations, pyTranslations, idsWithICU, md5WithIds);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
var translationsByView = {};
for (var id in translations) {
var ids = id.split(separator); // [viewName, stringId]
var viewName = ids[0];
var stringId = ids[1];
if (!translationsByView.hasOwnProperty(viewName)) {
translationsByView[viewName] = {};
}
if (!translationsByView[viewName].hasOwnProperty(lang)) {
translationsByView[viewName][lang] = {};
}
translationsByView[viewName][lang][stringId] = translations[id];
}
return translationsByView;
};
/**
* Grabs the translated strings from the po files for the given language and strings
* @param {str} lang iso code of the language to use
* @param {object} idsWithICU key: '<viewName>-<react-intl string id>'.
* value: english strings for translation
* @param {object} md5WithIds key: md5 hash of the english strings for translation.
* value: '<viewName>-<react-intl string id>'
* @return {object} translations – sub-objects by view containing:
* key: '<react-intl string id>'
* value: translated version of string
* empty if no translation present
*/
Helpers.getTranslationsForLanguageWithoutDefaults = function (lang, idsWithICU, md5WithIds, separator) {
var poUiDir = path.resolve(__dirname, '../../node_modules/scratchr2_translations/ui');
var jsFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/djangojs.po');
var pyFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/django.po');
var translations = {};
separator = separator || ':';
try {
fs.accessSync(jsFile, fs.R_OK);
var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
translations = Helpers.mergeNewTranslationsWithoutDefaults(translations,
jsTranslations, idsWithICU, md5WithIds);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
try {
fs.accessSync(pyFile, fs.R_OK);
var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
translations = Helpers.mergeNewTranslationsWithoutDefaults(translations,
pyTranslations, idsWithICU, md5WithIds);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
var translationsByView = {};
for (var id in translations) {
var ids = id.split(separator); // [viewName, stringId]
var viewName = ids[0];
var stringId = ids[1];
if (!translationsByView.hasOwnProperty(viewName)) {
translationsByView[viewName] = {};
}
if (!translationsByView[viewName].hasOwnProperty(lang)) {
translationsByView[viewName][lang] = {};
}
translationsByView[viewName][lang][stringId] = translations[id];
}
return translationsByView;
};
Helpers.writeTranslationsToJS = function (outputDir, viewName, translationObject) {
var objectString = JSON.stringify(translationObject);
var fileString = 'window._messages = ' + objectString + ';';
fs.writeFileSync(outputDir + '/' + viewName + '.intl.js', fileString);
};
// Returns a FormattedMessage id with english string as value. Use for default values in translations
// Sample structure: { 'general-general.blah': 'blah', 'about-about.blah': 'blahblah' }
Helpers.idToICUMap = function (viewName, ids, separator) {
var idsToICU = {};
separator = separator || ':';
for (var id in ids) {
idsToICU[viewName + separator + id] = ids[id];
}
return idsToICU;
};
// Reuturns reverse (i.e. english string with message key as the value) object for searching po files.
// Sample structure: { 'blah': 'general-general.blah', 'blahblah': 'about-about.blah' }
Helpers.icuToIdMap = function (viewName, ids, separator) {
var icuToIds = {};
separator = separator || ':';
for (var id in ids) {
icuToIds[ids[id]] = [viewName + separator + id];
}
return icuToIds;
};
Helpers.writeTranslations = function (name, l10n, languages) {
var ids = require(l10n);
var idsWithICU = Helpers.idToICUMap(name, ids);
var icuWithIds = Helpers.icuToIdMap(name, ids);
var md5WithIds = Helpers.getMD5Map(icuWithIds);
var isoCodes = Object.keys(languages);
var outputDir = path.resolve(__dirname, '../../localizations/', name);
try {
fs.accessSync(outputDir, fs.F_OK);
} catch (err) {
// Doesn't exist - create it.
fs.mkdirSync(outputDir);
}
process.stdout.write(`Writing translations to ${outputDir}\n`);
for (var isoCode in isoCodes) {
if (isoCodes[isoCode] !== 'en'){
var translations = Helpers.getTranslationsForLanguageWithoutDefaults(
isoCodes[isoCode], idsWithICU, md5WithIds);
var fileString = JSON.stringify(translations[name][isoCodes[isoCode]]);
fs.writeFileSync(outputDir + '/' + isoCodes[isoCode] + '.json', fileString);
}
}
};
module.exports = Helpers;

119
bin/tx-import Executable file
View file

@ -0,0 +1,119 @@
#!/usr/bin/env node
/*
Downloads translation json files from Transifex for all the languages defined
in the menus. Expects a Transifex API key to be set in the TX_TOKEN environment
variable. Takes as input a directory in which to store the translations.
Creates subdirectories for each view and 'general'. Creates <lang>.json files
for each language downloaded. Uses the english source l10n file if there is
an error.
*/
var fs = require('fs');
var path = require('path');
var async = require('async');
var Transifex = require('transifex');
var languages = require('../languages.json');
var routes = require('../src/routes.json');
// -----------------------------------------------------------------------------
// Main script
// -----------------------------------------------------------------------------
var args = process.argv.slice(2);
if (!args.length) {
process.stdout.write('A destination directory for localizations must be specified.');
process.exit(1);
}
var outputDir = path.resolve(__dirname, '../', args.shift());
try {
fs.accessSync(outputDir, fs.F_OK);
} catch (err) {
// Doesn't exist - create it.
fs.mkdirSync(outputDir);
}
if (process.env.TX_TOKEN) {
var transifex = new Transifex({
project_slug: 'scratch-website',
credential: 'api:'+process.env.TX_TOKEN
});
} else {
process.stdout.write('WARNING: Missing Transifex API key, skipping download.\n');
process.stdout.write('Set TX_TOKEN in env if you want translations.\n');
process.exit(0);
}
//make sure general translation directory exists
var transDir = path.resolve(outputDir + '/general');
try {
fs.accessSync(transDir, fs.F_OK);
} catch (err) {
// Doesn't exist - create it.
fs.mkdirSync(transDir);
}
var localizations = [];
for ( var isoCode in languages ) {
var transFile = transDir + '/' + isoCode + '.json';
localizations.push({view: 'general', lang: isoCode, file: transFile, src: 'l10n.json'});
}
// initialize views, ignore redirect routes
for (var v in routes) {
if (typeof routes[v].redirect !== 'undefined') {
continue;
}
var subdir = routes[v].view.split('/');
subdir.pop();
var l10n = 'src/views/' + subdir.join('/') + '/l10n.json';
var name = routes[v].name;
try {
// only Initialize if there is an l10n file
fs.accessSync(l10n);
transDir = path.resolve(outputDir + '/' + name);
try {
fs.accessSync(transDir, fs.F_OK);
} catch (err) {
// Doesn't exist - create it.
fs.mkdirSync(transDir);
}
for ( var isoCode in languages ) {
var transFile = transDir + '/' + isoCode + '.json';
localizations.push({view: name, lang: isoCode, file: transFile, src: l10n});
}
} catch (err) {
// skip views without l10n files
// TODO: ES6
// process.stdout.write(`Skipping ${name}, no l10n\n`);
process.stdout.write('Skipping ' + name+ ', no l10n\n');
}
}
async.forEachLimit(localizations, 3, function (item, cb) {
transifex.translationInstanceMethod(
'scratch-website',
item.view+'-l10njson',
item.lang,
{mode: 'reviewed'},
function (err, data) {
if (err) {
process.stdout.write(item.view + ': ' + item.lang + '(' + err + ')' + ' using english\n');
fs.createReadStream(item.src).pipe(fs.createWriteStream(item.file));
} else {
fs.writeFile(item.file, data);
}
return;
});
cb();
}, function (err) {
if (err) {
process.stdout.write('Import completed with errors\n');
}
return;
});

6
languages.json
View file

@ -1,5 +1,4 @@
{
"ab": "Аҧсшәа",
"ar": "العربية",
"an": "Aragonés",
"ast": "Asturianu",
@ -11,9 +10,7 @@
"cs": "Česky",
"cy": "Cymraeg",
"da": "Dansk",
"fa-af": "Dari",
"de": "Deutsch",
"yum": "Edible Scratch",
"et": "Eesti",
"el": "Ελληνικά",
"en": "English",
@ -46,12 +43,10 @@
"ml": "മലയാളം",
"mt": "Malti",
"mr": "मराठी",
"cat": "Meow",
"mn": "Монгол хэл",
"my": "မြန်မာဘာသာ",
"nl": "Nederlands",
"ja": "日本語",
"ja-hr": "にほんご",
"nb": "Norsk Bokmål",
"nn": "Norsk Nynorsk",
"uz": "Oʻzbekcha",
@ -69,7 +64,6 @@
"fi": "Suomi",
"sv": "Svenska",
"te": "తెలుగు",
"nai": "Tepehuan",
"vi": "Tiếng Việt",
"tr": "Türkçe",
"uk": "Українська",

5
package.json
View file

@ -88,6 +88,7 @@
"source-map-support": "0.3.2",
"style-loader": "0.12.3",
"tap": "7.1.2",
"transifex": "1.4.6",
"url-loader": "0.5.6",
"watch": "0.16.0",
"webpack": "1.12.14",
@ -96,8 +97,8 @@
},
"nyc": {
"include": [
"bin/**/*.js",
"src/**/*.js"
"bin/**/*.js",
"src/**/*.js"
],
"all": true
}

4
test/fixtures/build_locales.json vendored
View file

@ -1,4 +0,0 @@
{
"<p> <img src=\"{STATIC_URL}/images/help/ask-and-wait.png\" /> asks a question and stores the keyboard input in <img src=\"{STATIC_URL}/images/help/answer.png\" />. The answer is shared by all sprites. </p><p>If you want to save the current answer, you can store it in a variable or list. For example, <img src=\"{STATIC_URL}/images/help/answer-ex2.png\"/> </p><p>To view the value of answer, click the checkbox next to the answer block.<br><img src=\"{STATIC_URL}/images/help/answer-checkbox.png\" /></p>": "test.id1",
"<p> <img src=\"{STATIC_URL}/images/help/ask-and-wait.png\" /> asks a question and stores the keyboard input in <img src=\"{STATIC_URL}/images/help/answer.png\" />. The question appears in a voice balloon on the screen. The program waits as the user types in a response, until the Enter key is pressed or the check mark is clicked. </p>": "test.id2"
}

72
test/fixtures/test_es.po vendored
View file

@ -1,72 +0,0 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2015-03-20 14:16+0000\n"
"PO-Revision-Date: 2015-09-21 17:37+0000\n"
"Last-Translator: Anonymous Pootle User\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
"Language: es\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"X-Generator: Pootle 2.5.1.1\n"
"X-POOTLE-MTIME: 1442857052.000000\n"
#: test.html:15
#, python-format
msgid ""
"\n"
"<p> <img src=\"%(STATIC_URL)s/images/help/ask-and-wait.png\" /> asks a "
"question and stores the keyboard input in <img src=\"%(STATIC_URL)s/images/"
"help/answer.png\" />. The answer is shared by all sprites. </p>\n"
"<p>If you want to save the current answer, you can store it in a variable or "
"list. For example, <img src=\"%(STATIC_URL)s/images/help/answer-ex2.png\"/"
"> \n"
"</p>\n"
"\n"
"<p>\n"
"To view the value of answer, click the checkbox next to the answer block."
"<br>\n"
"<img src=\"%(STATIC_URL)s/images/help/answer-checkbox.png\" />\n"
"</p>"
msgstr ""
"\n"
"<p><img src=\"%(STATIC_URL)s/images/help/es/ask-and-wait.png\" /> hace una "
"pregunta y almacena la entrada de teclado en <img src=\"%(STATIC_URL)s/"
"images/help/es/answer.png\" />. La respuesta se comparte para todos los "
"objetos. </p>\n"
"<p>Si deseas guardar la respuesta actual, debes almacenarla en una variable "
"o una lista. Por ejemplo, <img src=\"%(STATIC_URL)s/images/help/es/answer-"
"ex2.png\"/> \n"
"</p>\n"
"\n"
"<p>\n"
"Si deseas ver el valor de una respuesta, haz clic en la casilla que aparece "
"junto al bloque de respuesta.<br>\n"
"<img src=\"%(STATIC_URL)s/images/help/es/answer-checkbox.png\" />\n"
"</p>"
#: test.html:18
#, python-format
msgid ""
"\n"
"<p> <img src=\"%(STATIC_URL)s/images/help/ask-and-wait.png\" /> asks a "
"question and stores the keyboard input in <img src=\"%(STATIC_URL)s/images/"
"help/answer.png\" />. The question appears in a voice balloon on the screen. "
"The program waits as the user types in a response, until the Enter key is "
"pressed or the check mark is clicked. \n"
"</p>"
msgstr ""
"\n"
"<p> <img src=\"%(STATIC_URL)s/images/help/es/ask-and-wait.png\" /> hace una "
"pregunta y almacena la entrada de teclado en <img src=\"%(STATIC_URL)s/"
"images/help/es/answer.png\" />. La pregunta aparece en un globo de voz en la "
"pantalla. El programa espera hasta que el usuario escriba una respuesta y "
"presione Enter o haga clic en la casilla de aprobación.\n"
"</p>"

4
test/fixtures/test_es_md5map.json vendored
View file

@ -1,4 +0,0 @@
{
"2ec20d41b181e1a41c071e13f414a74d": ["test.id1"],
"37ba6d5ef524504215f478912155f9ba": ["test.id2"]
}

16
test/functional/build_locales_complex_strings.js
View file

@ -1,16 +0,0 @@
var fs = require('fs');
var path = require('path');
var po2icu = require('po2icu');
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesFile', function (t) {
var md5map = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/test_es_md5map.json'), 'utf8'));
var newTranslations = po2icu.poFileToICUSync('es', path.resolve(__dirname, '../fixtures/test_es.po'));
var translations = buildLocales.mergeNewTranslations({}, newTranslations, {}, md5map);
t.ok(translations['test.id1'] !== undefined);
t.ok(translations['test.id2'] !== undefined);
t.end();
});

23
test/functional/build_locales_icuMap.js
View file

@ -1,23 +0,0 @@
var merge = require('lodash.merge');
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildIcuMap', function (t) {
var ids1 = {
'first.test1' : 'We know where we\'re going, but we don\'t know where we\'ve been',
'first.test2' : 'You may find yourself living in a shotgun shack'
};
var ids2 = {
'second.test1' : 'We know where we\'re going, but we don\'t know where we\'ve been',
'second.test2' : 'Gadji beri bimba clandridi'
};
var testicuMap = buildLocales.icuToIdMap('first', ids1);
testicuMap = merge(testicuMap, buildLocales.icuToIdMap('second', ids2), buildLocales.customMerge);
t.ok(testicuMap['We know where we\'re going, but we don\'t know where we\'ve been'].length === 2);
t.ok(testicuMap['You may find yourself living in a shotgun shack'].length === 1);
t.ok(testicuMap['Gadji beri bimba clandridi'].length === 1);
t.end();
});

16
test/functional/build_locales_md5map.js
View file

@ -1,16 +0,0 @@
var fs = require('fs');
var path = require('path');
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesFile', function (t) {
var actualMd5map = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/test_es_md5map.json'), 'utf8'));
var templates = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/build_locales.json'), 'utf8'));
var testMd5map = buildLocales.getMD5Map(templates);
for (var key in actualMd5map) {
t.ok(testMd5map[key] !== undefined);
}
t.end();
});

25
test/functional/build_locales_mergeDupStrings.js
View file

@ -1,25 +0,0 @@
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesMergeDupStrings', function (t) {
var existingTranslations = {
'test.test1': 'It\'s like raaayaaain, on your wedding day',
'test.test2': 'Free to flyyy, when you already paid',
'test.test4': 'Free to flyyy, when you already paid'
};
var newTranslations = {
'Isn\'t it ironic? No.': 'Es irónico? No.'
};
var md5map = {
'c21ce5ceefe167028182032d4255a384': ['test.test1'],
'9c40648034e467e16f8d6ae24bd610ab': ['test.test2', 'test.test4'],
'6885a345adafb3a9dd43d9f549430c88': ['test.test3', 'test.test5']
};
var mergedTranslations = buildLocales.mergeNewTranslations(existingTranslations, newTranslations, {}, md5map);
t.ok(mergedTranslations['test.test2'] !== undefined);
t.ok(mergedTranslations['test.test3'] !== undefined);
t.ok(mergedTranslations['test.test5'] !== undefined);
t.end();
});

23
test/functional/build_locales_mergeTranslations.js
View file

@ -1,23 +0,0 @@
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesMergeTranslations', function (t) {
var existingTranslations = {
'test.test1': 'It\'s like raaayaaain, on your wedding day',
'test.test2': 'Free to flyyy, when you already paid'
};
var newTranslations = {
'Isn\'t it ironic? No.': 'Es irónico? No.'
};
var md5map = {
'c21ce5ceefe167028182032d4255a384': ['test.test1'],
'9c40648034e467e16f8d6ae24bd610ab': ['test.test2'],
'6885a345adafb3a9dd43d9f549430c88': ['test.test3']
};
var mergedTranslations = buildLocales.mergeNewTranslations(existingTranslations, newTranslations, {}, md5map);
t.ok(mergedTranslations['test.test3'] !== undefined);
t.ok(mergedTranslations['test.test2'] !== undefined);
t.end();
});

33
test/localization/spot_check_about_has_strings.js
View file

@ -1,33 +0,0 @@
/*
* spot check that each language has values for the string id keys on About page
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckAboutStrings', function (t) {
var isoCodes = Object.keys(languages);
isoCodes.splice(isoCodes.indexOf('en'), 1);
var ids = require(path.resolve(__dirname, '../../src/views/about/l10n.json'));
var viewLocales = {
about: {en: ids}
};
var idsWithICU = localeCompare.idToICUMap('about', ids);
var icuWithIds = localeCompare.icuToIdMap('about', ids);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['about']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(translations['about'][isoCodes[i]]).sort(),
keysToCheck,
'check About keys for language ' + isoCodes[i]
);
}
t.end();
});

33
test/localization/spot_check_cards_has_strings.js
View file

@ -1,33 +0,0 @@
/*
* spot check that each language has values for the string id keys on Cards page
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckCardStrings', function (t) {
var isoCodes = Object.keys(languages);
isoCodes.splice(isoCodes.indexOf('en'), 1);
var ids = require(path.resolve(__dirname, '../../src/views/cards/l10n.json'));
var viewLocales = {
cards: {en: ids}
};
var idsWithICU = localeCompare.idToICUMap('cards', ids);
var icuWithIds = localeCompare.icuToIdMap('cards', ids);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['cards']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(translations['cards'][isoCodes[i]]).sort(),
keysToCheck,
'check Cards keys for language ' + isoCodes[i]
);
}
t.end();
});

33
test/localization/spot_check_faq_has_strings.js
View file

@ -1,33 +0,0 @@
/*
* spot check that each language has values for the string id keys on FAQ page
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckFaqStrings', function (t) {
var isoCodes = Object.keys(languages);
isoCodes.splice(isoCodes.indexOf('en'), 1);
var ids = require(path.resolve(__dirname, '../../src/views/faq/l10n.json'));
var viewLocales = {
faq: {en: ids}
};
var idsWithICU = localeCompare.idToICUMap('faq', ids);
var icuWithIds = localeCompare.icuToIdMap('faq', ids);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['faq']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(translations['faq'][isoCodes[i]]).sort(),
keysToCheck,
'check About keys for language ' + isoCodes[i]
);
}
t.end();
});

33
test/localization/spot_check_general_has_strings.js
View file

@ -1,33 +0,0 @@
/*
* spot check that each language has values for the string id keys used generally in the site
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckGeneralStrings', function (t) {
var isoCodes = Object.keys(languages);
isoCodes.splice(isoCodes.indexOf('en'), 1);
var ids = require(path.resolve(__dirname, '../../src/l10n.json'));
var viewLocales = {
general: {en: ids}
};
var idsWithICU = localeCompare.idToICUMap('general', ids);
var icuWithIds = localeCompare.icuToIdMap('general', ids);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['general']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(translations['general'][isoCodes[i]]).sort(),
keysToCheck,
'check About keys for language ' + isoCodes[i]
);
}
t.end();
});

72
test/localization/spot_check_nav.js
View file

@ -1,72 +0,0 @@
/*
* spot checks the translation of the nav bar for a select set
* of languages that cover a number of types of translations.
*
* Languages checked:
* - Hebrew
* - Edible Scratch (fake language)
* - Mandarin
* - Japanese
* - Brasilian Portuguese
* - Polish
* - Norwegian
* - German
*/
var path = require('path');
var tap = require('tap');
var localeCompare = require('../../bin/lib/locale-compare');
var languagesToCheck = [
'he', 'zh-cn', 'ja', 'pt-br', 'pl', 'nb'
];
var idsToCheck = [
'general.about', 'general.create', 'general.help', 'general.joinScratch',
'general.signIn', 'general.discuss'
];
var ids = require(path.resolve(__dirname, '../../src/l10n.json'));
var viewLocales = {
general: {en: ids}
};
var idsWithICU = localeCompare.idToICUMap('general', ids);
var icuWithIds = localeCompare.icuToIdMap('general', ids);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
tap.test('spotCheckNavBar', function (t) {
for (var i in languagesToCheck) {
var translations = localeCompare.getTranslationsForLanguage(languagesToCheck[i], idsWithICU, md5WithIds);
for (var j in idsToCheck) {
t.notEqual(
translations['general'][languagesToCheck[i]][idsToCheck[j]],
viewLocales['general']['en'][idsToCheck[j]],
'check localization of ' + idsToCheck[j] + ' for ' + languagesToCheck[i]
);
}
}
t.end();
});
// Test splash items for fake language.
var fakeLanguageIdsToCheck = ['news.scratchNews', 'splash.featuredProjects', 'splash.featuredStudios'];
ids = require(path.resolve(__dirname, '../../src/views/splash/l10n.json'));
viewLocales = {
splash: {en: ids}
};
idsWithICU = localeCompare.idToICUMap('splash', ids);
icuWithIds = localeCompare.icuToIdMap('splash', ids);
md5WithIds = localeCompare.getMD5Map(icuWithIds);
tap.test('spotCheckNavBarFakeLanguage', function (t) {
var translations = localeCompare.getTranslationsForLanguage('yum', idsWithICU, md5WithIds);
for (var i in fakeLanguageIdsToCheck) {
t.notEqual(
translations['splash']['yum'][fakeLanguageIdsToCheck[i]],
viewLocales['splash']['en'][fakeLanguageIdsToCheck[i]],
'check localization of ' + fakeLanguageIdsToCheck[i] + ' for yum'
);
}
t.end();
});

33
test/localization/spot_check_splash_has_strings.js
View file

@ -1,33 +0,0 @@
/*
* spot check that each language has values for the string id keys on Splash page
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckSplashStrings', function (t) {
var isoCodes = Object.keys(languages);
isoCodes.splice(isoCodes.indexOf('en'), 1);
var ids = require(path.resolve(__dirname, '../../src/views/splash/l10n.json'));
var viewLocales = {
splash: {en: ids}
};
var idsWithICU = localeCompare.idToICUMap('splash', ids);
var icuWithIds = localeCompare.icuToIdMap('splash', ids);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['splash']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(translations['splash'][isoCodes[i]]).sort(),
keysToCheck,
'check Splash keys for language ' + isoCodes[i]
);
}
t.end();
});

33
test/localization/spot_check_wedo2_has_strings.js
View file

@ -1,33 +0,0 @@
/*
* spot check that each language has values for the string id keys on Wedo2 page
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckWedo2Strings', function (t) {
var isoCodes = Object.keys(languages);
isoCodes.splice(isoCodes.indexOf('en'), 1);
var ids = require(path.resolve(__dirname, '../../src/views/wedo2/l10n.json'));
var viewLocales = {
wedo2: {en: ids}
};
var idsWithICU = localeCompare.idToICUMap('wedo2', ids);
var icuWithIds = localeCompare.icuToIdMap('wedo2', ids);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['wedo2']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(translations['wedo2'][isoCodes[i]]).sort(),
keysToCheck,
'check About keys for language ' + isoCodes[i]
);
}
t.end();
});

11
test/unit/build_locales_getmd5.js
View file

@ -1,11 +0,0 @@
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesGetMD5', function (t) {
var testString1 = 'are there bears here?';
var testString2 = 'are\nthere\tbears here?';
t.equal(buildLocales.getMD5(testString1), buildLocales.getMD5(testString2));
t.end();
});