Merge pull request #223 from mewtaylor/cleanup/locale-updates

Clean up `build-locales` and add tests for it.

Makefile

@@ -1,6 +1,7 @@
 ESLINT=./node_modules/.bin/eslint
 NODE=node
 SASSLINT=./node_modules/.bin/sass-lint -v
+TAP=./node_modules/.bin/tap
 WATCH=./node_modules/.bin/watch
 WEBPACK=./node_modules/.bin/webpack
 
@@ -34,7 +35,7 @@ static:
 	cp -a ./static/. ./build/
 
 translations:
-	./src/scripts/build-locales locales/translations.json
+	./lib/bin/build-locales locales/translations.json
 
 webpack:
 	$(WEBPACK) --bail
@@ -59,6 +60,11 @@ start:
 test:
 	@make lint
 	@make build
+	@echo ""
+	@make unit
+	@echo ""
+	@make functional
+	@echo ""
 
 lint:
 	$(ESLINT) ./*.js
@@ -72,6 +78,15 @@ lint:
 	$(SASSLINT) ./src/views/**/*.scss
 	$(SASSLINT) ./src/components/**/*.scss
 
+unit:
+	$(TAP) ./test/unit/*.js
+
+functional:
+	$(TAP) ./test/functional/*.js
+
+integration:
+	$(TAP) ./test/integration/*.js
+
 # ------------------------------------
 
 .PHONY: build clean deploy static translations webpack watch stop start test lint

lib/bin/build-locales

@@ -5,29 +5,17 @@
     Requires po2json in order to work. Takes as input a directory
     in which to store the resulting json translation files.
 */
-
 var fs = require('fs');
 var glob = require('glob');
 var path = require('path');
 var po2icu = require('po2icu');
 
-/*
-    Existing translations should be in the key value format specified by react-intl (i.e.
-    formatted message id, with icu string as the value). New Translations should be in the
-    format returned by po2icu (i.e. a source language icu string for key, and a localized
-    language icu string for value).
+var localeCompare = require('../locale-compare');
+
+// -----------------------------------------------------------------------------
+// Main script
+// -----------------------------------------------------------------------------
 
-    ICU Map is an object in the reverse react-intl formatting (icu string as key), which will
-    help determine if the translation belongs in www currently.
-*/
-var mergeNewTranslations = function (existingTranslations, newTranslations, icuMap) {
-    for (var id in newTranslations) {
-        if (icuMap.hasOwnProperty(id) && newTranslations[id].length > 0) {
-            existingTranslations[icuMap[id]] = newTranslations[id];
-        }
-    }
-    return existingTranslations;
-};
 
 var args = process.argv.slice(2);
 
@@ -49,13 +37,15 @@ try {
 
 var icuTemplateFile = path.resolve(__dirname, '../../en.json');
 var idsWithICU = JSON.parse(fs.readFileSync(icuTemplateFile, 'utf8'));
+var locales = {
+    en: idsWithICU
+};
+
 var icuWithIds = {};
 for (var id in idsWithICU) {
     icuWithIds[idsWithICU[id]] = id;
 }
-var locales = {
-    en: idsWithICU
-};
+var md5WithIds = localeCompare.getMD5Map(icuWithIds);
 
 // Get ui localization strings first
 glob(poUiDir + '/*', function (err, files) {
@@ -69,14 +59,14 @@ glob(poUiDir + '/*', function (err, files) {
         var translations = {};
         try {
             var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
-            translations = mergeNewTranslations(translations, jsTranslations, icuWithIds);
+            translations = localeCompare.mergeNewTranslations(translations, jsTranslations, md5WithIds);
         } catch (err) {
             process.stdout.write(lang + ': ' + err + '\n');
         }
 
         try {
             var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
-            translations = mergeNewTranslations(translations, pyTranslations, icuWithIds);
+            translations = localeCompare.mergeNewTranslations(translations, pyTranslations, md5WithIds);
         } catch (err) {
             process.stdout.write(lang + ': ' + err + '\n');
         }

lib/locale-compare.js

@@ -0,0 +1,60 @@
+// -----------------------------------------------------------------------------
+// Helper Methods for build-locales node script.
+// -----------------------------------------------------------------------------
+
+var crypto = require('crypto');
+
+var Helpers = {};
+
+/**
+ * Get the md5 has of a string with whitespace removed.
+ *
+ * @param {string} string a string
+ * @return {string} an md5 hash of the string in hex.
+ */
+Helpers.getMD5 = function (string) {
+    var cleanedString = string.replace(/\s+/g, '');
+    return crypto.createHash('md5').update(cleanedString, 'utf8').digest('hex');
+};
+
+/*
+    Existing translations should be in the key value format specified by react-intl (i.e.
+    formatted message id, with icu string as the value). New Translations should be in the
+    format returned by po2icu (i.e. a source language icu string for key, and a localized
+    language icu string for value).
+
+    ICU Map is an object in the reverse react-intl formatting (icu string as key), which will
+    help determine if the translation belongs in www currently.
+*/
+Helpers.mergeNewTranslations = function (existingTranslations, newTranslations, md5Map) {
+    for (var id in newTranslations) {
+        var md5 = Helpers.getMD5(id);
+        if (md5Map.hasOwnProperty(md5) && newTranslations[id].length > 0) {
+            existingTranslations[md5Map[md5]] = newTranslations[id];
+        }
+    }
+    return existingTranslations;
+};
+
+/**
+ * Converts a map of icu strings with react-intl id values into a map
+ * with md5 hashes of the icu strings as keys and react-intl id values.
+ * This is done so as to eliminate potential po conversion misses that
+ * could be caused by different white space formatting between po and icu.
+ *
+ * The md5 is generated after all white space is removed from the string.
+ *
+ * @param   {object}    idICUMap    map where key=icuString, value=react-intl id
+ *
+ * @return  {object}
+ */
+Helpers.getMD5Map = function (ICUIdMap) {
+    var md5Map = {};
+    for (var icu in ICUIdMap) {
+        var md5 = Helpers.getMD5(icu);
+        md5Map[md5] = ICUIdMap[icu];
+    }
+    return md5Map;
+};
+
+module.exports = Helpers;

test/fixtures/build_locales.json

@@ -0,0 +1,4 @@
+{
+    "<p> <img src=\"{STATIC_URL}/images/help/ask-and-wait.png\" /> asks a question and stores the keyboard input in  <img src=\"{STATIC_URL}/images/help/answer.png\" />. The answer is shared by all sprites. </p><p>If you want to save the current answer, you can store it in a variable or list. For example, <img src=\"{STATIC_URL}/images/help/answer-ex2.png\"/> </p><p>To view the value of answer, click the checkbox next to the answer block.<br><img src=\"{STATIC_URL}/images/help/answer-checkbox.png\" /></p>": "test.id1",
+    "<p> <img src=\"{STATIC_URL}/images/help/ask-and-wait.png\" /> asks a question and stores the keyboard input in  <img src=\"{STATIC_URL}/images/help/answer.png\" />. The question appears in a voice balloon on the screen. The program waits as the user types in a response, until the Enter key is pressed or the check mark is clicked. </p>": "test.id2"
+}

test/fixtures/test_es.po

@@ -0,0 +1,72 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2015-03-20 14:16+0000\n"
+"PO-Revision-Date: 2015-09-21 17:37+0000\n"
+"Last-Translator: Anonymous Pootle User\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: es\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+"X-Generator: Pootle 2.5.1.1\n"
+"X-POOTLE-MTIME: 1442857052.000000\n"
+
+#: test.html:15
+#, python-format
+msgid ""
+"\n"
+"<p> <img src=\"%(STATIC_URL)s/images/help/ask-and-wait.png\" /> asks a "
+"question and stores the keyboard input in  <img src=\"%(STATIC_URL)s/images/"
+"help/answer.png\" />. The answer is shared by all sprites. </p>\n"
+"<p>If you want to save the current answer, you can store it in a variable or "
+"list. For example, <img src=\"%(STATIC_URL)s/images/help/answer-ex2.png\"/"
+"> \n"
+"</p>\n"
+"\n"
+"<p>\n"
+"To view the value of answer, click the checkbox next to the answer block."
+"<br>\n"
+"<img src=\"%(STATIC_URL)s/images/help/answer-checkbox.png\" />\n"
+"</p>"
+msgstr ""
+"\n"
+"<p><img src=\"%(STATIC_URL)s/images/help/es/ask-and-wait.png\" /> hace una "
+"pregunta y almacena la entrada de teclado en <img src=\"%(STATIC_URL)s/"
+"images/help/es/answer.png\" />. La respuesta se comparte para todos los "
+"objetos. </p>\n"
+"<p>Si deseas guardar la respuesta actual, debes almacenarla en una variable "
+"o una lista. Por ejemplo, <img src=\"%(STATIC_URL)s/images/help/es/answer-"
+"ex2.png\"/> \n"
+"</p>\n"
+"\n"
+"<p>\n"
+"Si deseas ver el valor de una respuesta, haz clic en la casilla que aparece "
+"junto al bloque de respuesta.<br>\n"
+"<img src=\"%(STATIC_URL)s/images/help/es/answer-checkbox.png\" />\n"
+"</p>"
+
+#: test.html:18
+#, python-format
+msgid ""
+"\n"
+"<p> <img src=\"%(STATIC_URL)s/images/help/ask-and-wait.png\" /> asks a "
+"question and stores the keyboard input in  <img src=\"%(STATIC_URL)s/images/"
+"help/answer.png\" />. The question appears in a voice balloon on the screen. "
+"The program waits as the user types in a response, until the Enter key is "
+"pressed or the check mark is clicked. \n"
+"</p>"
+msgstr ""
+"\n"
+"<p> <img src=\"%(STATIC_URL)s/images/help/es/ask-and-wait.png\" /> hace una "
+"pregunta y almacena la entrada de teclado en <img src=\"%(STATIC_URL)s/"
+"images/help/es/answer.png\" />. La pregunta aparece en un globo de voz en la "
+"pantalla. El programa espera hasta que el usuario escriba una respuesta y "
+"presione Enter o haga clic en la casilla de aprobación.\n"
+"</p>"

test/fixtures/test_es_md5map.json

@@ -0,0 +1,4 @@
+{
+    "2ec20d41b181e1a41c071e13f414a74d": "test.id1",
+    "37ba6d5ef524504215f478912155f9ba": "test.id2"
+}

test/functional/build_locales_complex_strings.js

@@ -0,0 +1,16 @@
+var fs = require('fs');
+var path = require('path');
+var po2icu = require('po2icu');
+var tap = require('tap');
+
+var buildLocales = require('../../lib/locale-compare');
+
+tap.test('buildLocalesFile', function (t) {
+    var md5map = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/test_es_md5map.json'), 'utf8'));
+    var newTranslations = po2icu.poFileToICUSync('es', path.resolve(__dirname, '../fixtures/test_es.po'));
+    var translations = buildLocales.mergeNewTranslations({}, newTranslations, md5map);
+
+    t.ok(translations['test.id1'] !== undefined);
+    t.ok(translations['test.id2'] !== undefined);
+    t.end();
+});

test/functional/build_locales_md5map.js

@@ -0,0 +1,16 @@
+var fs = require('fs');
+var path = require('path');
+var tap = require('tap');
+
+var buildLocales = require('../../lib/locale-compare');
+
+tap.test('buildLocalesFile', function (t) {
+    var actualMd5map = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/test_es_md5map.json'), 'utf8'));
+    var templates = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/build_locales.json'), 'utf8'));
+    var testMd5map = buildLocales.getMD5Map(templates);
+
+    for (var key in actualMd5map) {
+        t.ok(testMd5map[key] !== undefined);
+    }
+    t.end();
+});

test/functional/build_locales_mergeTranslations.js

@@ -0,0 +1,23 @@
+var tap = require('tap');
+
+var buildLocales = require('../../lib/locale-compare');
+
+tap.test('buildLocalesMergeTranslations', function (t) {
+    var existingTranslations = {
+        'test.test1': 'It\'s like raaayaaain, on your wedding day',
+        'test.test2': 'Free to flyyy, when you already paid'
+    };
+    var newTranslations = {
+        'Isn\'t it ironic? No.': 'Es irónico? No.'
+    };
+    var md5map = {
+        'c21ce5ceefe167028182032d4255a384': 'test.test1',
+        '9c40648034e467e16f8d6ae24bd610ab': 'test.test2',
+        '6885a345adafb3a9dd43d9f549430c88': 'test.test3'
+    };
+
+    var mergedTranslations = buildLocales.mergeNewTranslations(existingTranslations, newTranslations, md5map);
+    t.ok(mergedTranslations['test.test3'] !== undefined);
+    t.ok(mergedTranslations['test.test2'] !== undefined);
+    t.end();
+});

test/unit/build_locales_getmd5.js

@@ -0,0 +1,11 @@
+var tap = require('tap');
+
+var buildLocales = require('../../lib/locale-compare');
+
+tap.test('buildLocalesGetMD5', function (t) {
+    var testString1 = 'are there bears here?';
+    var testString2 = 'are\nthere\tbears here?';
+    
+    t.equal(buildLocales.getMD5(testString1), buildLocales.getMD5(testString2));
+    t.end();
+});