mirror of
https://github.com/scratchfoundation/scratch-l10n.git
synced 2024-12-22 05:32:34 -05:00
Scripts to pull translations and update Freshdesk
* Fixed a bug in the push script (tags were not saved as correct structured json, so they weren’t getting put into transifex) * extended FreshDesk API with functions to update knowledge base. Functions automatically try to create the item if it isn’t found for updating * tx-pull-help-names: pulls category and folder name translations from transifex and updates them in Freshdesk (Note, since we don’t send people into the knowledge base, these aren’t really public, but the actual folders and categories have to exist to be able to save articles) * tx-pull-help-articles: pull article translations from Transifex and update the Freshdesk KB. * help-utils: utility functions for the tx-pull-help-* scripts. Handles limiting the number of things happening in parallel - currently two languages may be processed at the same time.
This commit is contained in:
Chris Garrity
parent
cea399a1f9
commit
30b1975e94
5 changed files with 362 additions and 2 deletions
|
|
@ -10,6 +10,7 @@ class FreshdeskApi {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
'Authorization': this._auth
|
'Authorization': this._auth
|
||||||
};
|
};
|
||||||
|
this.rateLimited = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -24,7 +25,9 @@ class FreshdeskApi {
|
||||||
}
|
}
|
||||||
throw new Error(`response not json: ${res.headers.get('content-type')}`);
|
throw new Error(`response not json: ${res.headers.get('content-type')}`);
|
||||||
}
|
}
|
||||||
throw new Error(`response: ${res.statusText}`);
|
let err = new Error(`response ${res.statusText}`);
|
||||||
|
err.code = res.status;
|
||||||
|
throw err;
|
||||||
}
|
}
|
||||||
|
|
||||||
listCategories () {
|
listCategories () {
|
||||||
|
|
@ -48,6 +51,105 @@ class FreshdeskApi {
|
||||||
.then(this.checkStatus)
|
.then(this.checkStatus)
|
||||||
.then(res => res.json());
|
.then(res => res.json());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
updateCategoryTranslation (id, locale, body) {
|
||||||
|
if (this.rateLimited) {
|
||||||
|
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return fetch(
|
||||||
|
`${this.baseUrl}/api/v2/solutions/categories/${id}/${locale}`,
|
||||||
|
{
|
||||||
|
method: 'put',
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
headers: this.defaultHeaders
|
||||||
|
})
|
||||||
|
.then(this.checkStatus)
|
||||||
|
.then(res => res.json())
|
||||||
|
.catch((err) => {
|
||||||
|
if (err.code === 404) {
|
||||||
|
// not found, try create instead
|
||||||
|
return fetch(
|
||||||
|
`${this.baseUrl}/api/v2/solutions/categories/${id}/${locale}`,
|
||||||
|
{
|
||||||
|
method: 'post',
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
headers: this.defaultHeaders
|
||||||
|
})
|
||||||
|
.then(this.checkStatus)
|
||||||
|
.then(res => res.json());
|
||||||
|
}
|
||||||
|
// re-raise the error otherwise
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
updateFolderTranslation (id, locale, body) {
|
||||||
|
if (this.rateLimited) {
|
||||||
|
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return fetch(
|
||||||
|
`${this.baseUrl}/api/v2/solutions/folders/${id}/${locale}`,
|
||||||
|
{
|
||||||
|
method: 'put',
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
headers: this.defaultHeaders
|
||||||
|
})
|
||||||
|
.then(this.checkStatus)
|
||||||
|
.then(res => res.json())
|
||||||
|
.catch((err) => {
|
||||||
|
if (err.code === 404) {
|
||||||
|
// not found, try create instead
|
||||||
|
return fetch(
|
||||||
|
`${this.baseUrl}/api/v2/solutions/folders/${id}/${locale}`,
|
||||||
|
{
|
||||||
|
method: 'post',
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
headers: this.defaultHeaders
|
||||||
|
})
|
||||||
|
.then(this.checkStatus)
|
||||||
|
.then(res => res.json());
|
||||||
|
}
|
||||||
|
// re-raise the error otherwise
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
updateArticleTranslation (id, locale, body) {
|
||||||
|
if (this.rateLimited) {
|
||||||
|
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return fetch(
|
||||||
|
`${this.baseUrl}/api/v2/solutions/articles/${id}/${locale}`,
|
||||||
|
{
|
||||||
|
method: 'put',
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
headers: this.defaultHeaders
|
||||||
|
})
|
||||||
|
.then(this.checkStatus)
|
||||||
|
.then(res => res.json())
|
||||||
|
.catch((err) => {
|
||||||
|
if (err.code === 404) {
|
||||||
|
// not found, try create instead
|
||||||
|
return fetch(
|
||||||
|
`${this.baseUrl}/api/v2/solutions/articles/${id}/${locale}`,
|
||||||
|
{
|
||||||
|
method: 'post',
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
headers: this.defaultHeaders
|
||||||
|
})
|
||||||
|
.then(this.checkStatus)
|
||||||
|
.then(res => res.json());
|
||||||
|
}
|
||||||
|
if (err.code === 429) {
|
||||||
|
this.rateLimited = true;
|
||||||
|
}
|
||||||
|
// re-raise the error otherwise
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = FreshdeskApi;
|
module.exports = FreshdeskApi;
|
||||||
|
|
|
||||||
189
scripts/help-utils.js
Normal file
189
scripts/help-utils.js
Normal file
|
|
@ -0,0 +1,189 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @fileoverview
|
||||||
|
* Helper functions for syncing Freshdesk knowledge base articles with Transifex
|
||||||
|
*/
|
||||||
|
|
||||||
|
const transifex = require('transifex');
|
||||||
|
const FreshdeskApi = require('./freshdesk-api.js');
|
||||||
|
const util = require('util');
|
||||||
|
// const fs = require('fs');
|
||||||
|
// const mkdirp = require('mkdirp');
|
||||||
|
|
||||||
|
const FD = new FreshdeskApi('https://mitscratch.freshdesk.com', process.env.FRESHDESK_TOKEN);
|
||||||
|
const TX_PROJECT = 'scratch-help';
|
||||||
|
const TX = new transifex({
|
||||||
|
project_slug: TX_PROJECT,
|
||||||
|
credential: 'api:' + process.env.TX_TOKEN
|
||||||
|
});
|
||||||
|
|
||||||
|
const freshdeskLocale = locale => {
|
||||||
|
// map between Transifex locale and Freshdesk. Two letter codes are usually fine
|
||||||
|
const localeMap = {
|
||||||
|
'es_419': 'es-LA',
|
||||||
|
'ja': 'ja-JP',
|
||||||
|
'ja-Hira': 'ja-JP',
|
||||||
|
'lv': 'lv-LV',
|
||||||
|
'nb': 'nb-NO',
|
||||||
|
'nn': 'nb-NO',
|
||||||
|
'pt': 'pt-PT',
|
||||||
|
'pt_BR': 'pt-BR',
|
||||||
|
'ru': 'ru-RU',
|
||||||
|
'sv': 'sv-SE',
|
||||||
|
'zh_CN': 'zh-CN',
|
||||||
|
'zh_TW': 'zh-TW'
|
||||||
|
};
|
||||||
|
return localeMap[locale] || locale;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Promisify Transifex async/callbacks to make them easier to use`
|
||||||
|
const getResources = util.promisify(TX.resourcesSetMethod.bind(TX));
|
||||||
|
const getResourceInfo = util.promisify(TX.resourcesInstanceMethods.bind(TX));
|
||||||
|
const getTranslation = util.promisify(TX.translationInstanceMethod.bind(TX));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pull metadata from Transifex for the scratch-help project
|
||||||
|
* @return {Promise} results array containing:
|
||||||
|
* languages: array of supported languages
|
||||||
|
* folders: array of tx resources corrsponding to Freshdesk folders
|
||||||
|
* names: array of tx resources corresponding to the Freshdesk metadata
|
||||||
|
*/
|
||||||
|
exports.getInputs = async () => {
|
||||||
|
const resources = await getResources(TX_PROJECT);
|
||||||
|
// there are three types of resources differentiated by the file type
|
||||||
|
const folders = resources.filter(resource => resource.i18n_type === 'STRUCTURED_JSON');
|
||||||
|
const names = resources.filter(resource => resource.i18n_type === 'KEYVALUEJSON');
|
||||||
|
// ignore the yaml type because it's not possible to update via API
|
||||||
|
|
||||||
|
// Lookup available languages by getting metadata for a resource, they all have the
|
||||||
|
// same set of languages, so it doesn't matter which one you get
|
||||||
|
const resourceInfo = await getResourceInfo(TX_PROJECT, resources[0].slug, true);
|
||||||
|
const languages = resourceInfo.available_languages.map(l => l.code);
|
||||||
|
|
||||||
|
return Promise.all([languages, folders, names]); // eslint-disable-line no-undef
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* internal function to serialize saving category and folder name translations to avoid Freshdesk rate limit
|
||||||
|
* @param {[type]} json [description]
|
||||||
|
* @param {[type]} resource [description]
|
||||||
|
* @param {[type]} locale [description]
|
||||||
|
* @return {Promise} [description]
|
||||||
|
*/
|
||||||
|
const serializeNameSave = async (json, resource, locale) => {
|
||||||
|
for (let [key, value] of Object.entries(json)) {
|
||||||
|
// key is of the form <name>_<id>
|
||||||
|
const words = key.split('_');
|
||||||
|
const id = words[words.length - 1];
|
||||||
|
let status = 0;
|
||||||
|
if (resource.name === 'categoryNames_json') {
|
||||||
|
status = await FD.updateCategoryTranslation(id, freshdeskLocale(locale), {name: value});
|
||||||
|
}
|
||||||
|
if (resource.name === 'folderNames_json') {
|
||||||
|
status = await FD.updateFolderTranslation(id, freshdeskLocale(locale), {name: value});
|
||||||
|
}
|
||||||
|
if (status === -1) {
|
||||||
|
process.exitCode = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Internal function serialize Freshdesk requests to avoid getting rate limited
|
||||||
|
* @param {object} json object with keys corresponding to article ids
|
||||||
|
* @param {string} locale language code
|
||||||
|
* @return {Promise} [description]
|
||||||
|
*/
|
||||||
|
const serializeFolderSave = async (json, locale) => {
|
||||||
|
// json is a map of articles:
|
||||||
|
// {
|
||||||
|
// <id>: {
|
||||||
|
// title: {string: <title-value>},
|
||||||
|
// description: {string: <description-value>},
|
||||||
|
// tags: {string: <comma separated strings} // optional
|
||||||
|
// },
|
||||||
|
// <id>: {
|
||||||
|
// title: {string: <title-value>},
|
||||||
|
// description: {string: <description-value>},
|
||||||
|
// tags: {string: <comma separated strings} // optional
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
for (let [id, value] of Object.entries(json)) {
|
||||||
|
let body = {
|
||||||
|
title: value.title.string,
|
||||||
|
description: value.description.string,
|
||||||
|
status: 2 // set status to published
|
||||||
|
};
|
||||||
|
if (value.hasOwnProperty('tags')) {
|
||||||
|
body.tags = value.tags.string.split(',');
|
||||||
|
}
|
||||||
|
let status = await FD.updateArticleTranslation(id, freshdeskLocale(locale), body);
|
||||||
|
if (status === -1) {
|
||||||
|
process.exitCode = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process Transifex resource corresponding to a Knowledge base folder on Freshdesk
|
||||||
|
* @param {object} folder Transifex resource json corresponding to a KB folder
|
||||||
|
* @param {string} locale locale to pull and submit to Freshdesk
|
||||||
|
* @return {Promise} [description]
|
||||||
|
*/
|
||||||
|
exports.localizeFolder = async (folder, locale) => {
|
||||||
|
getTranslation(TX_PROJECT, folder.slug, locale, {mode: 'default'})
|
||||||
|
.then(data => {
|
||||||
|
const json = JSON.parse(data);
|
||||||
|
|
||||||
|
serializeFolderSave(json, locale);
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
process.stdout.write(`Error processing ${folder.slug}, ${locale}: ${e.message}\n`);
|
||||||
|
process.exitCode = 1; // not ok
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process KEYVALUEJSON resources from scratch-help on transifex
|
||||||
|
* Category and Folder names are stored as plain json
|
||||||
|
* @param {object} resource Transifex resource json for either CategoryNames or FolderNames
|
||||||
|
* @param {string} locale locale to pull and submit to Freshdesk
|
||||||
|
* @return {Promise} [description]
|
||||||
|
*/
|
||||||
|
exports.localizeNames = async (resource, locale) => {
|
||||||
|
getTranslation(TX_PROJECT, resource.slug, locale, {mode: 'default'})
|
||||||
|
.then(data => {
|
||||||
|
const json = JSON.parse(data);
|
||||||
|
serializeNameSave(json, resource, locale);
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
process.stdout.write(`Error saving ${resource.slug}, ${locale}: ${e.message}\n`);
|
||||||
|
process.exitCode = 1; // not ok
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
const BATCH_SIZE = 2;
|
||||||
|
/*
|
||||||
|
* save resource items in batches to reduce rate limiting errors
|
||||||
|
* @param {object} item Transifex resource json, used for 'slug'
|
||||||
|
* @param {array} languages Array of languages to save
|
||||||
|
* @param {function} saveFn Async function to use to save the item
|
||||||
|
* @return {Promise}
|
||||||
|
*/
|
||||||
|
exports.saveItem = async (item, languages, saveFn) => {
|
||||||
|
const saveLanguages = languages.filter(l => l !== 'en'); // exclude English from update
|
||||||
|
let batchedPromises = Promise.resolve(); // eslint-disable-line no-undef
|
||||||
|
for (let i = 0; i < saveLanguages.length; i += BATCH_SIZE) {
|
||||||
|
batchedPromises = batchedPromises
|
||||||
|
.then(() => Promise.all( // eslint-disable-line
|
||||||
|
saveLanguages.slice(i, i + BATCH_SIZE).map(l => saveFn(item, l))
|
||||||
|
))
|
||||||
|
.catch(err => {
|
||||||
|
process.stdout.write(`Error saving item:${err.message}\n${JSON.stringify(item, null, 2)}\n`);
|
||||||
|
process.exitCode = 1; // not ok
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
34
scripts/tx-pull-help-articles.js
Executable file
34
scripts/tx-pull-help-articles.js
Executable file
|
|
@ -0,0 +1,34 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @fileoverview
|
||||||
|
* Script to pull scratch-help translations from transifex and push to FreshDesk.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const usage = `
|
||||||
|
Pull knowledge base articles from transifex and push to FreshDesk. Usage:
|
||||||
|
node tx-pull-help.js
|
||||||
|
NOTE:
|
||||||
|
FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with
|
||||||
|
access to the Knowledge Base.
|
||||||
|
TX_TOKEN environment variable needs to be set with a Transifex API token. See
|
||||||
|
the Localization page on the GUI wiki for information about setting up Transifex.
|
||||||
|
`;
|
||||||
|
// Fail immediately if the API tokens are not defined, or there any argument
|
||||||
|
if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
|
||||||
|
process.stdout.write(usage);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const {getInputs, saveItem, localizeFolder} = require('./help-utils.js');
|
||||||
|
|
||||||
|
getInputs()
|
||||||
|
.then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars
|
||||||
|
process.stdout.write('Processing articles pulled from Transifex\n');
|
||||||
|
return folders.map(item => saveItem(item, languages, localizeFolder));
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
process.stdout.write(`Error: ${e.message}\n`);
|
||||||
|
process.exitCode = 1; // not ok
|
||||||
|
});
|
||||||
35
scripts/tx-pull-help-names.js
Executable file
35
scripts/tx-pull-help-names.js
Executable file
|
|
@ -0,0 +1,35 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @fileoverview
|
||||||
|
* Script to pull scratch-help translations from transifex and push to FreshDesk.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
|
||||||
|
const usage = `
|
||||||
|
Pull knowledge base category and folder names from transifex and push to FreshDesk. Usage:
|
||||||
|
node tx-pull-help.js
|
||||||
|
NOTE:
|
||||||
|
FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with
|
||||||
|
access to the Knowledge Base.
|
||||||
|
TX_TOKEN environment variable needs to be set with a Transifex API token. See
|
||||||
|
the Localization page on the GUI wiki for information about setting up Transifex.
|
||||||
|
`;
|
||||||
|
// Fail immediately if the API tokens are not defined, or there any argument
|
||||||
|
if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
|
||||||
|
process.stdout.write(usage);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const {getInputs, saveItem, localizeNames} = require('./help-utils.js');
|
||||||
|
|
||||||
|
getInputs()
|
||||||
|
.then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars
|
||||||
|
process.stdout.write('Process Category and Folder Names pulled from Transifex\n');
|
||||||
|
return names.map(item => saveItem(item, languages, localizeNames));
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
process.stdout.write(`Error: ${e.message}\n`);
|
||||||
|
process.exitCode = 1; // not ok
|
||||||
|
});
|
||||||
|
|
@ -102,7 +102,7 @@ const saveArticles = (folder) => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if (current.tags.length > 0) {
|
if (current.tags.length > 0) {
|
||||||
strings[`${current.id}`].tags = current.tags.toString();
|
strings[`${current.id}`].tags = {string: current.tags.toString()};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return strings;
|
return strings;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue