Use transifex .download() to download

Undo tr change

lib/batch.js

@@ -0,0 +1,17 @@
+/**
+ * Maps each value of an array into an async function, and returns an array of the results
+ * @param {array} arr - array of values
+ * @param {number} batchSize - number of calls to `func` to do at one time
+ * @param {function} func - async function to apply to all items in `arr`. Function should take one argument.
+ * @return {array} - results of `func` applied to each item in `arr`
+ */
+exports.batchMap = async (arr, batchSize, func) => {
+    const results = [];
+    for (let i = 0; i < arr.length; i += batchSize) {
+        const result = await Promise.all( // eslint-disable-line
+            arr.slice(i, i + batchSize).map(func)
+        );
+        results.push(...result);
+    }
+    return results;
+};

lib/transifex.js

@@ -6,8 +6,8 @@
  * TODO: add functions for pushing to Transifex
  */
 
-import {transifexApi} from '@transifex/api';
-import download from 'download';
+const transifexApi = require('@transifex/api').transifexApi;
+const download = require('download');
 
 const ORG_NAME = 'llk';
 const SOURCE_LOCALE = 'en';
@@ -23,74 +23,15 @@ try {
     throw err;
 }
 
-const timeout = async function (ms) {
-    return new Promise(r => setTimeout(r, ms)); // eslint-disable-line no-undef
-};
-
-/**
- * Wait until a download is ready, and then get the location of the file to download.
- * In order to do this, we need to check the download status until it 303s. Once it 303s,
- * we can get the download location from the header.
- * @param {*} downloadEventId - id of the transifex download event
- * @param {*} isSourceLocale - whether the locale is the same as the source locale
- * @returns {string} - url of file that is ready to download
- */
-const getDownloadFileLocation = async function (downloadEventId, isSourceLocale) {
-    const statusEndpoint = isSourceLocale ?
-        transifexApi.ResourceStringAsyncDownload :
-        transifexApi.ResourceTranslationAsyncDownload;
-
-    let waitMs = 100;
-
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-        try {
-            const downloadStatus = await statusEndpoint.get(downloadEventId);
-
-            if (downloadStatus.attributes.status === 'failed') {
-                if (downloadStatus.attributes.errors.length() > 0) {
-                    throw new Error(downloadStatus.attributes.errors);
-                }
-                throw new Error('Error downloading file');
-            }
-
-            /** If there is no error from getting the status, the download is still pending.
-            Wait before trying again. **/
-            await timeout(waitMs);
-
-            // exponentially increase wait time
-            waitMs = waitMs * 2;
-        } catch (err) {
-            /** status 303 means the download event is complete and the
-            resource can be downloaded at the location uri. **/
-            if (err.response.status === 303) {
-                return err.response.headers.location;
-            }
-            throw err;
-        }
-    }
-};
-
-/**
- * Given a download event id, waits for the event to complete and then downloads the file
- * @param {string} downloadEventId - id of the transifex download event
- * @param {boolean} isSourceLocale - whether the locale is the same as the source locale
- * @returns {object} - buffer of downloaded file
- */
-const downloadFile = async function (downloadEventId, isSourceLocale) {
-    const location = await getDownloadFileLocation(downloadEventId, isSourceLocale);
-    return await download(location);
-};
-
 /**
  * Creates a download event for a specific project, resource, and locale.
  * @param {string} projectSlug - project slug (for example,  "scratch-editor")
  * @param {string} resourceSlug - resource slug (for example,  "blocks")
  * @param {string} localeCode - language code (for example,  "ko")
- * @param {string} mode - translation status of strings to include (defaults to "reviewed")
+ * @param {string} mode - translation status of strings to include
  * @returns {string} - id of the created download event
  */
-const createDownloadEvent = async function (projectSlug, resourceSlug, localeCode, mode) {
+const downloadResource = async function (projectSlug, resourceSlug, localeCode, mode = 'default') {
     const resource = {
         data: {
             id: `o:${ORG_NAME}:p:${projectSlug}:r:${resourceSlug}`,
@@ -100,12 +41,9 @@ const createDownloadEvent = async function (projectSlug, resourceSlug, localeCod
 
     // if locale is English, create a download event of the source file
     if (localeCode === SOURCE_LOCALE) {
-        const sourceDownloadEvent = await transifexApi.ResourceStringAsyncDownload.create({
-            attributes: {file_type: 'default'},
-            relationships: {resource}
+        return await transifexApi.ResourceStringsAsyncDownload.download({
+            resource
         });
-
-        return sourceDownloadEvent.id;
     }
 
     const language = {
@@ -116,12 +54,11 @@ const createDownloadEvent = async function (projectSlug, resourceSlug, localeCod
     };
 
     // if locale is not English, create a download event of the translation file
-    const downloadEvent = await transifexApi.ResourceTranslationAsyncDownload.create({
-        attributes: {mode: mode},
-        relationships: {resource, language}
+    return await transifexApi.ResourceTranslationsAsyncDownload.download({
+        mode,
+        resource,
+        language
     });
-   
-    return downloadEvent.id;
 };
 
 /**
@@ -134,16 +71,9 @@ const createDownloadEvent = async function (projectSlug, resourceSlug, localeCod
  * strings, if the local is the source language)
  */
 const txPull = async function (project, resource, locale, mode = 'default') {
-    try {
-        const downloadId = await createDownloadEvent(project, resource, locale, mode);
-        const buffer = await downloadFile(downloadId, locale === SOURCE_LOCALE);
-        return JSON.parse(buffer.toString());
-    } catch (err) {
-        if (err.statusCode === 409) {
-            throw new Error({statusCode: 409, message: 'translation does not exist for ' + locale});
-        }
-        throw err;
-    }
+    const url = await downloadResource(project, resource, locale, mode);
+    const buffer = await download(url);
+    return JSON.parse(buffer.toString());
 };
 
 /**
@@ -166,4 +96,4 @@ const txResources = async function (project) {
     return slugs;
 };
 
-export {txPull, txResources};
+module.exports = {txPull, txResources};

package.json

@@ -48,7 +48,7 @@
     "dependencies": {
         "@babel/cli": "^7.1.2",
         "@babel/core": "^7.1.2",
-        "@transifex/api": "^2.4.0",
+        "@transifex/api": "3.0.0",
         "babel-plugin-react-intl": "^3.0.1",
         "download": "^8.0.0",
         "transifex": "1.6.6"

scripts/tx-pull-editor.js

@@ -29,9 +29,9 @@ if (!process.env.TX_TOKEN || args.length < 3) {
 import fs from 'fs';
 import path from 'path';
 import {txPull} from '../lib/transifex.js';
-import async from 'async';
 import {validateTranslations} from '../lib/validate.js';
 import locales, {localeMap} from '../src/supported-locales.js';
+import {batchMap} from '../lib/batch.js';
 
 // Globals
 const PROJECT = args[0];
@@ -43,7 +43,6 @@ const CONCURRENCY_LIMIT = 36;
 const getLocaleData = async function (locale) {
     let txLocale = localeMap[locale] || locale;
     const data = await txPull(PROJECT, RESOURCE, txLocale, MODE);
-    
     return {
         locale: locale,
         translations: data
@@ -52,8 +51,8 @@ const getLocaleData = async function (locale) {
 
 const pullTranslations = async function () {
     try {
-        const values = await async.mapLimit(Object.keys(locales), CONCURRENCY_LIMIT, getLocaleData);
-    
+        const values = await batchMap(Object.keys(locales), CONCURRENCY_LIMIT, getLocaleData);
+
         const source = values.find(elt => elt.locale === 'en').translations;
         values.forEach(function (translation) {
             validateTranslations({locale: translation.locale, translations: translation.translations}, source);
@@ -64,7 +63,7 @@ const pullTranslations = async function () {
             );
         });
     } catch (err) {
-        console.error(err); // eslint-disable-line no-console
+        process.stdout.write(err);
         process.exit(1);
     }
 };

scripts/tx-pull-www.js

@@ -31,14 +31,14 @@ import fs from 'fs';
 import path from 'path';
 import mkdirp from 'mkdirp';
 import {txPull, txResources} from '../lib/transifex.js';
-import async from 'async';
 import locales, {localeMap} from '../src/supported-locales.js';
+import {batchMap} from '../lib/batch.js';
 
 // Globals
 const PROJECT = 'scratch-website';
 const OUTPUT_DIR = path.resolve(args[0]);
 // const MODE = {mode: 'reviewed'}; // default is everything for www
-const CONCURRENCY_LIMIT = 4;
+const CONCURRENCY_LIMIT = 36;
 
 const lang = args.length === 2 ? args[1] : '';
 
@@ -82,7 +82,7 @@ const pullTranslations = async function () {
     const allFiles = expandResourceFiles(resources);
 
     try {
-        await async.mapLimit(allFiles, CONCURRENCY_LIMIT, getLocaleData);
+        await batchMap(allFiles, CONCURRENCY_LIMIT, getLocaleData);
     } catch (err) {
         console.error(err); // eslint-disable-line no-console
         process.exit(1);