fix: only retry the actual download
Some checks failed
CI/CD / build-and-deploy (push) Has been cancelled

Also, further improve error messaging
This commit is contained in:
Christopher Willis-Ford 2024-10-29 08:57:19 -07:00
parent 41c1466604
commit 905f004b15
2 changed files with 52 additions and 33 deletions

View file

@ -103,19 +103,34 @@ const getResourceLocation = async function (projectSlug, resourceSlug, localeCod
* strings, if the local is the source language) * strings, if the local is the source language)
*/ */
const txPull = async function (project, resource, locale, mode = 'default') { const txPull = async function (project, resource, locale, mode = 'default') {
const url = await getResourceLocation(project, resource, locale, mode);
let buffer; let buffer;
for (let i = 0; i < 5; i++) {
try { try {
buffer = await download(url); const url = await getResourceLocation(project, resource, locale, mode);
for (let i = 0; i < 5; i++) {
if (i > 0) {
console.log(`Retrying txPull download after ${i} failed attempt(s)`);
}
try {
buffer = await download(url); // might throw(?)
break;
} catch (e) {
console.error(e, {project, resource, locale, buffer});
}
}
if (!buffer) {
throw Error(`txPull download failed after 5 retries: ${url}`);
}
buffer = buffer.toString(); buffer = buffer.toString();
return JSON.parse(buffer); return JSON.parse(buffer);
} catch (e) { } catch (e) {
console.error(e, {project, resource, locale, buffer}); e.cause = {
console.log(`Retrying after ${i + 1} failed attempt(s)`); project,
resource,
locale,
buffer
};
throw e;
} }
}
throw Error('failed to pull after 5 retries');
}; };
/** /**

View file

@ -47,30 +47,34 @@ const getLocaleData = async function (item) {
const locale = item.locale; const locale = item.locale;
const resource = item.resource; const resource = item.resource;
let txLocale = localeMap[locale] || locale; let txLocale = localeMap[locale] || locale;
for (let i = 0; i < 5; i++) {
try {
const translations = await txPull(PROJECT, resource, txLocale); const translations = await txPull(PROJECT, resource, txLocale);
const txOutdir = `${OUTPUT_DIR}/${PROJECT}.${resource}`; const txOutdir = `${OUTPUT_DIR}/${PROJECT}.${resource}`;
mkdirp.sync(txOutdir);
const fileName = `${txOutdir}/${locale}.json`; const fileName = `${txOutdir}/${locale}.json`;
try {
mkdirp.sync(txOutdir);
await fs.writeFile( await fs.writeFile(
fileName, fileName,
JSON.stringify(translations, null, 4) JSON.stringify(translations, null, 4)
); );
this.progress.increment();
return { return {
resource: resource, resource,
locale: locale, locale,
file: fileName fileName
}; };
} catch (e) { } catch (e) {
console.error(e); e.cause = {
console.log(`retrying after ${i + 1} attempt(s)`); resource,
locale,
translations,
txOutdir,
fileName
};
throw e;
} }
}
throw Error('failed to pull translations after 5 retries');
}; };
const expandResourceFiles = (resources) => { const expandResourceFiles = (resources) => {
@ -94,9 +98,9 @@ const pullTranslations = async function () {
const progress = new ProgressLogger(allFiles.length); const progress = new ProgressLogger(allFiles.length);
try { try {
await batchMap(allFiles, CONCURRENCY_LIMIT, item => { await batchMap(allFiles, CONCURRENCY_LIMIT, async item => {
try { try {
getLocaleData(item); await getLocaleData(item);
} finally { } finally {
progress.increment(); progress.increment();
} }