scratchfoundation/scratch-l10n
1
0
Fork 0
mirror of https://github.com/scratchfoundation/scratch-l10n.git synced 2024-12-22 13:42:30 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Scripts to pull translations and update Freshdesk

Browse source 
* Fixed a bug in the push script (tags were not saved as correct structured json, so they weren’t getting put into transifex)
* extended FreshDesk API with functions to update knowledge base. Functions automatically try to create the item if it isn’t found for updating
* tx-pull-help-names: pulls category and folder name translations from transifex and updates them in Freshdesk (Note, since we don’t send people into the knowledge base, these aren’t really public, but the actual folders and categories have to exist to be able to save articles)
* tx-pull-help-articles: pull article translations from Transifex and update the Freshdesk KB.
* help-utils: utility functions for the tx-pull-help-* scripts. Handles limiting the number of things happening in parallel - currently two languages may be processed at the same time.
This commit is contained in:
Chris Garrity 2020-05-06 14:44:46 -04:00
parent cea399a1f9
commit 30b1975e94
5 changed files with 362 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

104
scripts/freshdesk-api.js
View file

@ -10,6 +10,7 @@ class FreshdeskApi {
'Content-Type': 'application/json',
'Authorization': this._auth
};
this.rateLimited = false;
}
/**
@ -24,7 +25,9 @@ class FreshdeskApi {
}
throw new Error(`response not json: ${res.headers.get('content-type')}`);
}
throw new Error(`response: ${res.statusText}`);
let err = new Error(`response ${res.statusText}`);
err.code = res.status;
throw err;
}
listCategories () {
@ -48,6 +51,105 @@ class FreshdeskApi {
.then(this.checkStatus)
.then(res => res.json());
}
updateCategoryTranslation (id, locale, body) {
if (this.rateLimited) {
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
return -1;
}
return fetch(
`${this.baseUrl}/api/v2/solutions/categories/${id}/${locale}`,
{
method: 'put',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json())
.catch((err) => {
if (err.code === 404) {
// not found, try create instead
return fetch(
`${this.baseUrl}/api/v2/solutions/categories/${id}/${locale}`,
{
method: 'post',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json());
}
// re-raise the error otherwise
throw err;
});
}
updateFolderTranslation (id, locale, body) {
if (this.rateLimited) {
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
return -1;
}
return fetch(
`${this.baseUrl}/api/v2/solutions/folders/${id}/${locale}`,
{
method: 'put',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json())
.catch((err) => {
if (err.code === 404) {
// not found, try create instead
return fetch(
`${this.baseUrl}/api/v2/solutions/folders/${id}/${locale}`,
{
method: 'post',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json());
}
// re-raise the error otherwise
throw err;
});
}
updateArticleTranslation (id, locale, body) {
if (this.rateLimited) {
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
return -1;
}
return fetch(
`${this.baseUrl}/api/v2/solutions/articles/${id}/${locale}`,
{
method: 'put',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json())
.catch((err) => {
if (err.code === 404) {
// not found, try create instead
return fetch(
`${this.baseUrl}/api/v2/solutions/articles/${id}/${locale}`,
{
method: 'post',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json());
}
if (err.code === 429) {
this.rateLimited = true;
}
// re-raise the error otherwise
throw err;
});
}
}
module.exports = FreshdeskApi;

189
scripts/help-utils.js Normal file
View file

@ -0,0 +1,189 @@
#!/usr/bin/env node
/**
* @fileoverview
* Helper functions for syncing Freshdesk knowledge base articles with Transifex
*/
const transifex = require('transifex');
const FreshdeskApi = require('./freshdesk-api.js');
const util = require('util');
// const fs = require('fs');
// const mkdirp = require('mkdirp');
const FD = new FreshdeskApi('https://mitscratch.freshdesk.com', process.env.FRESHDESK_TOKEN);
const TX_PROJECT = 'scratch-help';
const TX = new transifex({
project_slug: TX_PROJECT,
credential: 'api:' + process.env.TX_TOKEN
});
const freshdeskLocale = locale => {
// map between Transifex locale and Freshdesk. Two letter codes are usually fine
const localeMap = {
'es_419': 'es-LA',
'ja': 'ja-JP',
'ja-Hira': 'ja-JP',
'lv': 'lv-LV',
'nb': 'nb-NO',
'nn': 'nb-NO',
'pt': 'pt-PT',
'pt_BR': 'pt-BR',
'ru': 'ru-RU',
'sv': 'sv-SE',
'zh_CN': 'zh-CN',
'zh_TW': 'zh-TW'
};
return localeMap[locale] || locale;
};
// Promisify Transifex async/callbacks to make them easier to use`
const getResources = util.promisify(TX.resourcesSetMethod.bind(TX));
const getResourceInfo = util.promisify(TX.resourcesInstanceMethods.bind(TX));
const getTranslation = util.promisify(TX.translationInstanceMethod.bind(TX));
/**
* Pull metadata from Transifex for the scratch-help project
* @return {Promise} results array containing:
* languages: array of supported languages
* folders: array of tx resources corrsponding to Freshdesk folders
* names: array of tx resources corresponding to the Freshdesk metadata
*/
exports.getInputs = async () => {
const resources = await getResources(TX_PROJECT);
// there are three types of resources differentiated by the file type
const folders = resources.filter(resource => resource.i18n_type === 'STRUCTURED_JSON');
const names = resources.filter(resource => resource.i18n_type === 'KEYVALUEJSON');
// ignore the yaml type because it's not possible to update via API
// Lookup available languages by getting metadata for a resource, they all have the
// same set of languages, so it doesn't matter which one you get
const resourceInfo = await getResourceInfo(TX_PROJECT, resources[0].slug, true);
const languages = resourceInfo.available_languages.map(l => l.code);
return Promise.all([languages, folders, names]); // eslint-disable-line no-undef
};
/**
* internal function to serialize saving category and folder name translations to avoid Freshdesk rate limit
* @param {[type]} json [description]
* @param {[type]} resource [description]
* @param {[type]} locale [description]
* @return {Promise} [description]
*/
const serializeNameSave = async (json, resource, locale) => {
for (let [key, value] of Object.entries(json)) {
// key is of the form <name>_<id>
const words = key.split('_');
const id = words[words.length - 1];
let status = 0;
if (resource.name === 'categoryNames_json') {
status = await FD.updateCategoryTranslation(id, freshdeskLocale(locale), {name: value});
}
if (resource.name === 'folderNames_json') {
status = await FD.updateFolderTranslation(id, freshdeskLocale(locale), {name: value});
}
if (status === -1) {
process.exitCode = 1;
}
}
};
/**
* Internal function serialize Freshdesk requests to avoid getting rate limited
* @param {object} json object with keys corresponding to article ids
* @param {string} locale language code
* @return {Promise} [description]
*/
const serializeFolderSave = async (json, locale) => {
// json is a map of articles:
// {
// <id>: {
// title: {string: <title-value>},
// description: {string: <description-value>},
// tags: {string: <comma separated strings} // optional
// },
// <id>: {
// title: {string: <title-value>},
// description: {string: <description-value>},
// tags: {string: <comma separated strings} // optional
// }
// }
for (let [id, value] of Object.entries(json)) {
let body = {
title: value.title.string,
description: value.description.string,
status: 2 // set status to published
};
if (value.hasOwnProperty('tags')) {
body.tags = value.tags.string.split(',');
}
let status = await FD.updateArticleTranslation(id, freshdeskLocale(locale), body);
if (status === -1) {
process.exitCode = 1;
}
}
return 0;
};
/**
* Process Transifex resource corresponding to a Knowledge base folder on Freshdesk
* @param {object} folder Transifex resource json corresponding to a KB folder
* @param {string} locale locale to pull and submit to Freshdesk
* @return {Promise} [description]
*/
exports.localizeFolder = async (folder, locale) => {
getTranslation(TX_PROJECT, folder.slug, locale, {mode: 'default'})
.then(data => {
const json = JSON.parse(data);
serializeFolderSave(json, locale);
})
.catch((e) => {
process.stdout.write(`Error processing ${folder.slug}, ${locale}: ${e.message}\n`);
process.exitCode = 1; // not ok
});
};
/**
* Process KEYVALUEJSON resources from scratch-help on transifex
* Category and Folder names are stored as plain json
* @param {object} resource Transifex resource json for either CategoryNames or FolderNames
* @param {string} locale locale to pull and submit to Freshdesk
* @return {Promise} [description]
*/
exports.localizeNames = async (resource, locale) => {
getTranslation(TX_PROJECT, resource.slug, locale, {mode: 'default'})
.then(data => {
const json = JSON.parse(data);
serializeNameSave(json, resource, locale);
})
.catch((e) => {
process.stdout.write(`Error saving ${resource.slug}, ${locale}: ${e.message}\n`);
process.exitCode = 1; // not ok
});
};
const BATCH_SIZE = 2;
/*
* save resource items in batches to reduce rate limiting errors
* @param {object} item Transifex resource json, used for 'slug'
* @param {array} languages Array of languages to save
* @param {function} saveFn Async function to use to save the item
* @return {Promise}
*/
exports.saveItem = async (item, languages, saveFn) => {
const saveLanguages = languages.filter(l => l !== 'en'); // exclude English from update
let batchedPromises = Promise.resolve(); // eslint-disable-line no-undef
for (let i = 0; i < saveLanguages.length; i += BATCH_SIZE) {
batchedPromises = batchedPromises
.then(() => Promise.all( // eslint-disable-line
saveLanguages.slice(i, i + BATCH_SIZE).map(l => saveFn(item, l))
))
.catch(err => {
process.stdout.write(`Error saving item:${err.message}\n${JSON.stringify(item, null, 2)}\n`);
process.exitCode = 1; // not ok
});
}
};

34
scripts/tx-pull-help-articles.js Executable file
View file

@ -0,0 +1,34 @@
#!/usr/bin/env node
/**
* @fileoverview
* Script to pull scratch-help translations from transifex and push to FreshDesk.
*/
const args = process.argv.slice(2);
const usage = `
Pull knowledge base articles from transifex and push to FreshDesk. Usage:
node tx-pull-help.js
NOTE:
FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with
access to the Knowledge Base.
TX_TOKEN environment variable needs to be set with a Transifex API token. See
the Localization page on the GUI wiki for information about setting up Transifex.
`;
// Fail immediately if the API tokens are not defined, or there any argument
if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
process.stdout.write(usage);
process.exit(1);
}
const {getInputs, saveItem, localizeFolder} = require('./help-utils.js');
getInputs()
.then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars
process.stdout.write('Processing articles pulled from Transifex\n');
return folders.map(item => saveItem(item, languages, localizeFolder));
})
.catch((e) => {
process.stdout.write(`Error: ${e.message}\n`);
process.exitCode = 1; // not ok
});

35
scripts/tx-pull-help-names.js Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env node
/**
* @fileoverview
* Script to pull scratch-help translations from transifex and push to FreshDesk.
*/
const args = process.argv.slice(2);
const usage = `
Pull knowledge base category and folder names from transifex and push to FreshDesk. Usage:
node tx-pull-help.js
NOTE:
FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with
access to the Knowledge Base.
TX_TOKEN environment variable needs to be set with a Transifex API token. See
the Localization page on the GUI wiki for information about setting up Transifex.
`;
// Fail immediately if the API tokens are not defined, or there any argument
if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
process.stdout.write(usage);
process.exit(1);
}
const {getInputs, saveItem, localizeNames} = require('./help-utils.js');
getInputs()
.then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars
process.stdout.write('Process Category and Folder Names pulled from Transifex\n');
return names.map(item => saveItem(item, languages, localizeNames));
})
.catch((e) => {
process.stdout.write(`Error: ${e.message}\n`);
process.exitCode = 1; // not ok
});

2
scripts/tx-push-help.js
View file

@ -102,7 +102,7 @@ const saveArticles = (folder) => {
}
};
if (current.tags.length > 0) {
strings[`${current.id}`].tags = current.tags.toString();
strings[`${current.id}`].tags = {string: current.tags.toString()};
}
}
return strings;