scratchfoundation/scratch-www
1
0
Fork 0
mirror of https://github.com/scratchfoundation/scratch-www.git synced 2025-03-25 20:29:45 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions4

Refactor build-locales a bit

Browse source 
This makes a couple of procedure-specific parts into methods in `locale-compare`, so that they can be tested more readily.
This commit is contained in:
Matthew Taylor 2016-03-23 08:22:09 -04:00
parent ff003b137a
commit c448b15dd4
8 changed files with 228 additions and 107 deletions

5
Makefile
View file

@ -73,11 +73,6 @@ lint:
$(SASSLINT) ./src/views/**/*.scss
$(SASSLINT) ./src/components/**/*.scss
localization-standalone:
@make translations
@make localization
@echo ""
unit:
$(TAP) ./test/unit/*.js

98
bin/build-locales
View file

@ -39,8 +39,8 @@ var fs = require('fs');
var glob = require('glob');
var merge = require('lodash.merge');
var path = require('path');
var po2icu = require('po2icu');
var languages = require('../languages.json');
var localeCompare = require('./lib/locale-compare');
// -----------------------------------------------------------------------------
@ -55,7 +55,6 @@ if (!args.length) {
process.exit(1);
}
var poUiDir = path.resolve(__dirname, '../node_modules/scratchr2_translations/ui');
var outputDir = path.resolve(__dirname, '../', args[0]);
try {
fs.accessSync(outputDir, fs.F_OK);
@ -64,27 +63,19 @@ try {
fs.mkdirSync(outputDir);
}
// get global locale strings first.
var globalTemplateFile = path.resolve(__dirname, '../src/l10n.json');
// message key with english string values (i.e. default values)
var generalIds = JSON.parse(fs.readFileSync(globalTemplateFile, 'utf8'));
var viewLocales = {};
var generalLocales = {
en: generalIds
};
// FormattedMessage id with english string as value. Use for default values in translations
// Sample structure: { 'general-general.blah': 'blah', 'about-about.blah': 'blahblah' }
var idsWithICU = {};
// reverse (i.e. english string with message key as the value) object for searching po files.
// Sample structure: { 'blah': 'general-general.blah', 'blahblah': 'about-about.blah' }
var icuWithIds = {};
for (var id in generalIds) {
idsWithICU['general-' + id] = generalIds[id];
icuWithIds[generalIds[id]] = 'general-' + id;
}
// get global locale strings first.
var globalTemplateFile = path.resolve(__dirname, '../src/l10n.json');
localeCompare.getIdsForView('general', globalTemplateFile, viewLocales, idsWithICU, icuWithIds);
// start with all views, and remove localized ones as they are iterated over
var views = glob.sync(path.resolve(__dirname, '../src/views/*'));
@ -98,15 +89,7 @@ files.forEach(function (file) {
var dirPath = file.split('/');
dirPath.pop();
var view = dirPath.pop();
var viewIds = JSON.parse(fs.readFileSync(file, 'utf8'));
viewLocales[view] = {
en: viewIds
};
for (var id in viewIds) {
idsWithICU[view + '-' + id] = viewIds[id];
icuWithIds[viewIds[id]] = view + '-' + id; // add viewName to identifier for later
}
localeCompare.getIdsForView(view, file, viewLocales, idsWithICU, icuWithIds);
});
// md5 of english strings with message key as the value for searching po files.
@ -114,59 +97,18 @@ files.forEach(function (file) {
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
// Get ui localization strings first
glob(poUiDir + '/*', function (err, files) {
if (err) throw new Error(err);
files.forEach(function (file) {
var lang = file.split('/').pop();
var jsFile = path.resolve(file, 'LC_MESSAGES/djangojs.po');
var pyFile = path.resolve(file, 'LC_MESSAGES/django.po');
var translations = {};
try {
fs.accessSync(jsFile, fs.R_OK);
var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
translations = localeCompare.mergeNewTranslations(translations, jsTranslations, idsWithICU, md5WithIds);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
try {
fs.accessSync(pyFile, fs.R_OK);
var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
translations = localeCompare.mergeNewTranslations(translations, pyTranslations, idsWithICU, md5WithIds);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
// add new translations to locale object
for (var id in translations) {
var ids = id.split('-'); // [viewName, stringId]
var viewName = ids[0];
var stringId = ids[1];
if (viewLocales.hasOwnProperty(viewName)) {
if (!viewLocales[viewName].hasOwnProperty(lang)) viewLocales[viewName][lang] = {};
viewLocales[viewName][lang][stringId] = translations[id];
} else {
// default to general
if (!generalLocales.hasOwnProperty(lang)) generalLocales[lang] = {};
generalLocales[lang][stringId] = translations[id];
}
}
});
for (var i in views) {
var viewTranslations = generalLocales;
if (views[i] in viewLocales) {
viewTranslations = merge(viewLocales[views[i]], viewTranslations);
}
var objectString = JSON.stringify(viewTranslations);
var fileString = 'window._messages = ' + objectString + ';';
fs.writeFileSync(outputDir + '/' + views[i] + '.intl.js', fileString);
var isoCodes = Object.keys(languages);
for (i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
for (var key in translations) {
viewLocales[key] = merge(viewLocales[key], translations[key]);
}
});
}
for (i in views) {
var viewTranslations = viewLocales['general'];
if (views[i] in viewLocales) {
viewTranslations = merge(viewLocales[views[i]], viewTranslations);
}
localeCompare.writeTranslationsToJS(outputDir, views[i], viewTranslations);
}

74
bin/lib/locale-compare.js
View file

@ -3,6 +3,9 @@
// -----------------------------------------------------------------------------
var crypto = require('crypto');
var fs = require('fs');
var path = require('path');
var po2icu = require('po2icu');
var Helpers = {};
@ -62,4 +65,75 @@ Helpers.getMD5Map = function (ICUIdMap) {
return md5Map;
};
/**
* Grabs the translated strings from the po files for the given language and strings
* @param {str} lang iso code of the language to use
* @param {object} idsWithICU key: '<viewName>-<react-intl string id>'.
* value: english strings for translation
* @param {object} md5WithIds key: md5 hash of the english strings for translation.
* value: '<viewName>-<react-intl string id>'
* @return {object} translations – sub-objects by view containing:
* key: '<react-intl string id>'
* value: translated version of string
*/
Helpers.getTranslationsForLanguage = function (lang, idsWithICU, md5WithIds) {
var poUiDir = path.resolve(__dirname, '../../node_modules/scratchr2_translations/ui');
var jsFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/djangojs.po');
var pyFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/django.po');
var translations = {};
try {
fs.accessSync(jsFile, fs.R_OK);
var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
translations = Helpers.mergeNewTranslations(translations, jsTranslations, idsWithICU, md5WithIds);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
try {
fs.accessSync(pyFile, fs.R_OK);
var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
translations = Helpers.mergeNewTranslations(translations, pyTranslations, idsWithICU, md5WithIds);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
var translationsByView = {};
for (var id in translations) {
var ids = id.split('-'); // [viewName, stringId]
var viewName = ids[0];
var stringId = ids[1];
if (!translationsByView.hasOwnProperty(viewName)) {
translationsByView[viewName] = {};
}
if (!translationsByView[viewName].hasOwnProperty(lang)) {
translationsByView[viewName][lang] = {};
}
translationsByView[viewName][lang][stringId] = translations[id];
}
return translationsByView;
};
Helpers.writeTranslationsToJS = function (outputDir, viewName, translationObject) {
var objectString = JSON.stringify(translationObject);
var fileString = 'window._messages = ' + objectString + ';';
fs.writeFileSync(outputDir + '/' + viewName + '.intl.js', fileString);
};
Helpers.getIdsForView = function (viewName, viewFile, localeObject, idsWithICU, icuWithIds) {
var ids = JSON.parse(fs.readFileSync(viewFile, 'utf8'));
localeObject[viewName] = {
en: ids
};
for (var id in ids) {
idsWithICU[viewName + '-' + id] = ids[id];
icuWithIds[ids[id]] = viewName + '-' + id; // add viewName to identifier for later
}
};
module.exports = Helpers;

23
test/localization/spot_check_about_has_strings.js
View file

@ -2,17 +2,32 @@
* spot check that each language has values for the string id keys on About page
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
window = {};
require('../../intl/about.intl.js');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckAboutStrings', function (t) {
var isoCodes = Object.keys(languages);
var keysToCheck = Object.keys(window._messages['en']).sort();
isoCodes.splice(isoCodes.indexOf('en'), 1);
var viewLocales = {};
var idsWithICU = {};
var icuWithIds = {};
localeCompare.getIdsForView(
'about',
path.resolve(__dirname, '../../src/views/about/l10n.json'),
viewLocales,
idsWithICU,
icuWithIds
);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['about']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(window._messages[isoCodes[i]]).sort(),
Object.keys(translations['about'][isoCodes[i]]).sort(),
keysToCheck,
'check About keys for language ' + isoCodes[i]
);

36
test/localization/spot_check_general_has_strings.js Normal file
View file

@ -0,0 +1,36 @@
/*
* spot check that each language has values for the string id keys used generally in the site
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckAboutStrings', function (t) {
var isoCodes = Object.keys(languages);
isoCodes.splice(isoCodes.indexOf('en'), 1);
var viewLocales = {};
var idsWithICU = {};
var icuWithIds = {};
localeCompare.getIdsForView(
'general',
path.resolve(__dirname, '../../src/l10n.json'),
viewLocales,
idsWithICU,
icuWithIds
);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['general']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(translations['general'][isoCodes[i]]).sort(),
keysToCheck,
'check About keys for language ' + isoCodes[i]
);
}
t.end();
});

41
test/localization/spot_check_nav.js
View file

@ -12,9 +12,13 @@
* - Norwegian
* - German
*/
var path = require('path');
var tap = require('tap');
window = {};
require('../../intl/splash.intl.js');
var localeCompare = require('../../bin/lib/locale-compare');
var viewLocales = {};
var idsWithICU = {};
var icuWithIds = {};
var languagesToCheck = [
'he', 'zh-cn', 'ja', 'pt-br', 'pl', 'nb'
@ -24,12 +28,24 @@ var idsToCheck = [
'general.signIn', 'general.discuss'
];
// Test nav for real languages.
localeCompare.getIdsForView(
'general',
path.resolve(__dirname, '../../src/l10n.json'),
viewLocales,
idsWithICU,
icuWithIds
);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
tap.test('spotCheckNavBar', function (t) {
for (var i in languagesToCheck) {
var translations = localeCompare.getTranslationsForLanguage(languagesToCheck[i], idsWithICU, md5WithIds);
for (var j in idsToCheck) {
t.notEqual(
window._messages[languagesToCheck[i]][idsToCheck[j]],
window._messages['en'][idsToCheck[j]],
translations['general'][languagesToCheck[i]][idsToCheck[j]],
viewLocales['general']['en'][idsToCheck[j]],
'check localization of ' + idsToCheck[j] + ' for ' + languagesToCheck[i]
);
}
@ -37,12 +53,25 @@ tap.test('spotCheckNavBar', function (t) {
t.end();
});
// Test splash items for fake language.
var fakeLanguageIdsToCheck = ['news.scratchNews', 'splash.featuredProjects', 'splash.featuredStudios'];
localeCompare.getIdsForView(
'splash',
path.resolve(__dirname, '../../src/views/splash/l10n.json'),
viewLocales,
idsWithICU,
icuWithIds
);
md5WithIds = localeCompare.getMD5Map(icuWithIds);
tap.test('spotCheckNavBarFakeLanguage', function (t) {
var translations = localeCompare.getTranslationsForLanguage('yum', idsWithICU, md5WithIds);
for (var i in fakeLanguageIdsToCheck) {
t.notEqual(
window._messages['yum'][fakeLanguageIdsToCheck[i]],
window._messages['en'][fakeLanguageIdsToCheck[i]],
translations['general']['yum'][fakeLanguageIdsToCheck[i]],
viewLocales['splash']['en'][fakeLanguageIdsToCheck[i]],
'check localization of ' + fakeLanguageIdsToCheck[i] + ' for yum'
);
}

27
test/localization/spot_check_splash_has_strings.js
View file

@ -2,17 +2,32 @@
* spot check that each language has values for the string id keys on Splash page
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
window = {};
require('../../intl/splash.intl.js');
tap.test('spotCheckSplashStrings', function (t) {
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckAboutStrings', function (t) {
var isoCodes = Object.keys(languages);
var keysToCheck = Object.keys(window._messages['en']).sort();
isoCodes.splice(isoCodes.indexOf('en'), 1);
var viewLocales = {};
var idsWithICU = {};
var icuWithIds = {};
localeCompare.getIdsForView(
'splash',
path.resolve(__dirname, '../../src/views/splash/l10n.json'),
viewLocales,
idsWithICU,
icuWithIds
);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['splash']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(window._messages[isoCodes[i]]).sort(),
Object.keys(translations['splash'][isoCodes[i]]).sort(),
keysToCheck,
'check Splash keys for language ' + isoCodes[i]
);

31
test/localization/spot_check_wedo2_has_strings.js
View file

@ -1,20 +1,35 @@
/*
* spot check that each language has values for the string id keys on Splash page
* spot check that each language has values for the string id keys on Wedo2 page
* that are contained in English (i.e. make sure strings will show up, not ids")
*/
var merge = require('lodash.merge');
var path = require('path');
var tap = require('tap');
var languages = require('../../languages.json');
window = {};
require('../../intl/wedo2.intl.js');
tap.test('spotCheckWedo2Strings', function (t) {
var languages = require('../../languages.json');
var localeCompare = require('../../bin/lib/locale-compare');
tap.test('spotCheckAboutStrings', function (t) {
var isoCodes = Object.keys(languages);
var keysToCheck = Object.keys(window._messages['en']).sort();
isoCodes.splice(isoCodes.indexOf('en'), 1);
var viewLocales = {};
var idsWithICU = {};
var icuWithIds = {};
localeCompare.getIdsForView(
'wedo2',
path.resolve(__dirname, '../../src/views/wedo2/l10n.json'),
viewLocales,
idsWithICU,
icuWithIds
);
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var keysToCheck = Object.keys(merge(viewLocales['wedo2']['en'])).sort();
for (var i in isoCodes) {
var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
t.same(
Object.keys(window._messages[isoCodes[i]]).sort(),
Object.keys(translations['wedo2'][isoCodes[i]]).sort(),
keysToCheck,
'check Wedo2 keys for language ' + isoCodes[i]
'check About keys for language ' + isoCodes[i]
);
}
t.end();