scratchfoundation/scratch-l10n
1
0
Fork 0
mirror of https://github.com/scratchfoundation/scratch-l10n.git synced 2025-01-03 11:25:51 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #110 from chrisgarrity/freshdesk-scripts

Browse source 
Add scripts for syncing Knowledge base
This commit is contained in:
chrisgarrity 2020-05-27 14:43:33 -04:00 committed by GitHub
commit 78a90b9d6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 2885 additions and 342 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2581
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

8
package.json
View file

@ -21,6 +21,11 @@
"pull:blocks": "babel-node scripts/tx-pull-blocks ./editor/blocks/",
"pull:editor": "npm run pull:blocks && npm run pull:extensions && npm run pull:paint && npm run pull:interface",
"pull:www": "babel-node scripts/tx-pull-www ./www",
"pull:help:names": "./scripts/tx-pull-help-names.js",
"pull:help:articles": "./scripts/tx-pull-help-articles.js",
"pull:help": "npm run pull:help:names && npm run pull:help:articles",
"push:help": "./scripts/tx-push-help.js",
"sync:help": "npm run push:help && npm run pull:help",
"test": "npm run lint:js && npm run validate:editor && npm run validate:www && npm run build && npm run lint:json",
"update": "scripts/update-translations.sh",
"validate:blocks": "babel-node scripts/validate-translations ./editor/blocks/",
@ -51,7 +56,7 @@
"@babel/plugin-proposal-object-rest-spread": "^7.0.0",
"@babel/plugin-syntax-dynamic-import": "^7.0.0",
"@babel/plugin-transform-async-to-generator": "^7.1.0",
"@babel/preset-env": "^7.1.0",
"@babel/preset-env": "^7.9.5",
"@babel/preset-react": "^7.0.0",
"async": "2.6.1",
"babel-eslint": "^10.0.1",
@ -68,6 +73,7 @@
"jsonlint": "1.6.3",
"lodash.defaultsdeep": "4.6.0",
"mkdirp": "^0.5.1",
"node-fetch": "^2.6.0",
"p-limit": "2.1.0",
"p-queue": "3.0.0",
"react-intl": "2.8.0",

164
scripts/freshdesk-api.js Normal file
View file

@ -0,0 +1,164 @@
// interface to FreshDesk Solutions (knowledge base) api
const fetch = require('node-fetch');
class FreshdeskApi {
constructor (baseUrl, apiKey) {
this.baseUrl = baseUrl;
this._auth = 'Basic ' + new Buffer(`${apiKey}:X`).toString('base64');
this.defaultHeaders = {
'Content-Type': 'application/json',
'Authorization': this._auth
};
this.rateLimited = false;
}
/**
* Checks the status of a response. If status is not ok, or the body is not json raise exception
* @param {object} res The response object
* @returns {object} the response if it is ok
*/
checkStatus (res) {
if (res.ok) {
if (res.headers.get('content-type').indexOf('application/json') !== -1) {
return res;
}
throw new Error(`response not json: ${res.headers.get('content-type')}`);
}
let err = new Error(`response ${res.statusText}`);
err.code = res.status;
if (res.status === 429) {
err.retryAfter = res.headers.get('Retry-After');
}
throw err;
}
listCategories () {
return fetch(`${this.baseUrl}/api/v2/solutions/categories`, {headers: this.defaultHeaders})
.then(this.checkStatus)
.then(res => res.json());
}
listFolders (category) {
return fetch(
`${this.baseUrl}/api/v2/solutions/categories/${category.id}/folders`,
{headers: this.defaultHeaders})
.then(this.checkStatus)
.then(res => res.json());
}
listArticles (folder) {
return fetch(
`${this.baseUrl}/api/v2/solutions/folders/${folder.id}/articles`,
{headers: this.defaultHeaders})
.then(this.checkStatus)
.then(res => res.json());
}
updateCategoryTranslation (id, locale, body) {
if (this.rateLimited) {
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
return -1;
}
return fetch(
`${this.baseUrl}/api/v2/solutions/categories/${id}/${locale}`,
{
method: 'put',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json())
.catch((err) => {
if (err.code === 404) {
// not found, try create instead
return fetch(
`${this.baseUrl}/api/v2/solutions/categories/${id}/${locale}`,
{
method: 'post',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json());
}
if (err.code === 429) {
this.rateLimited = true;
}
process.stdout.write(`Error processing id ${id} for locale ${locale}: ${err.message}\n`);
throw err;
});
}
updateFolderTranslation (id, locale, body) {
if (this.rateLimited) {
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
return -1;
}
return fetch(
`${this.baseUrl}/api/v2/solutions/folders/${id}/${locale}`,
{
method: 'put',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json())
.catch((err) => {
if (err.code === 404) {
// not found, try create instead
return fetch(
`${this.baseUrl}/api/v2/solutions/folders/${id}/${locale}`,
{
method: 'post',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json());
}
if (err.code === 429) {
this.rateLimited = true;
}
process.stdout.write(`Error processing id ${id} for locale ${locale}: ${err.message}\n`);
throw err;
});
}
updateArticleTranslation (id, locale, body) {
if (this.rateLimited) {
process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`);
return -1;
}
return fetch(
`${this.baseUrl}/api/v2/solutions/articles/${id}/${locale}`,
{
method: 'put',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json())
.catch((err) => {
if (err.code === 404) {
// not found, try create instead
return fetch(
`${this.baseUrl}/api/v2/solutions/articles/${id}/${locale}`,
{
method: 'post',
body: JSON.stringify(body),
headers: this.defaultHeaders
})
.then(this.checkStatus)
.then(res => res.json());
}
if (err.code === 429) {
this.rateLimited = true;
}
process.stdout.write(`Error processing id ${id} for locale ${locale}: ${err.message}\n`);
throw err;
});
}
}
module.exports = FreshdeskApi;

217
scripts/help-utils.js Normal file
View file

@ -0,0 +1,217 @@
#!/usr/bin/env node
/**
* @fileoverview
* Helper functions for syncing Freshdesk knowledge base articles with Transifex
*/
const transifex = require('transifex');
const FreshdeskApi = require('./freshdesk-api.js');
const util = require('util');
const fs = require('fs');
const fsPromises = fs.promises;
const mkdirp = require('mkdirp');
const FD = new FreshdeskApi('https://mitscratch.freshdesk.com', process.env.FRESHDESK_TOKEN);
const TX_PROJECT = 'scratch-help';
const TX = new transifex({
project_slug: TX_PROJECT,
credential: 'api:' + process.env.TX_TOKEN
});
const freshdeskLocale = locale => {
// map between Transifex locale and Freshdesk. Two letter codes are usually fine
const localeMap = {
'es_419': 'es-LA',
'ja': 'ja-JP',
'ja-Hira': 'ja-JP',
'lv': 'lv-LV',
'nb': 'nb-NO',
'nn': 'nb-NO',
'pt': 'pt-PT',
'pt_BR': 'pt-BR',
'ru': 'ru-RU',
'sv': 'sv-SE',
'zh_CN': 'zh-CN',
'zh_TW': 'zh-TW'
};
return localeMap[locale] || locale;
};
// Promisify Transifex async/callbacks to make them easier to use`
const getResources = util.promisify(TX.resourcesSetMethod.bind(TX));
const getResourceInfo = util.promisify(TX.resourcesInstanceMethods.bind(TX));
const getTranslation = util.promisify(TX.translationInstanceMethod.bind(TX));
/**
* Pull metadata from Transifex for the scratch-help project
* @return {Promise} results array containing:
* languages: array of supported languages
* folders: array of tx resources corrsponding to Freshdesk folders
* names: array of tx resources corresponding to the Freshdesk metadata
*/
exports.getInputs = async () => {
const resources = await getResources(TX_PROJECT);
// there are three types of resources differentiated by the file type
const folders = resources.filter(resource => resource.i18n_type === 'STRUCTURED_JSON');
const names = resources.filter(resource => resource.i18n_type === 'KEYVALUEJSON');
// ignore the yaml type because it's not possible to update via API
// Lookup available languages by getting metadata for a resource, they all have the
// same set of languages, so it doesn't matter which one you get
const resourceInfo = await getResourceInfo(TX_PROJECT, resources[0].slug, true);
const languages = resourceInfo.available_languages.map(l => l.code);
return Promise.all([languages, folders, names]); // eslint-disable-line no-undef
};
/**
* internal function to serialize saving category and folder name translations to avoid Freshdesk rate limit
* @param {[type]} json [description]
* @param {[type]} resource [description]
* @param {[type]} locale [description]
* @return {Promise} [description]
*/
const serializeNameSave = async (json, resource, locale) => {
for (let [key, value] of Object.entries(json)) {
// key is of the form <name>_<id>
const words = key.split('_');
const id = words[words.length - 1];
let status = 0;
if (resource.name === 'categoryNames_json') {
status = await FD.updateCategoryTranslation(id, freshdeskLocale(locale), {name: value});
}
if (resource.name === 'folderNames_json') {
status = await FD.updateFolderTranslation(id, freshdeskLocale(locale), {name: value});
}
if (status === -1) {
process.exitCode = 1;
}
}
};
/**
* Internal function serialize Freshdesk requests to avoid getting rate limited
* @param {object} json object with keys corresponding to article ids
* @param {string} locale language code
* @return {Promise} [description]
*/
const serializeFolderSave = async (json, locale) => {
// json is a map of articles:
// {
// <id>: {
// title: {string: <title-value>},
// description: {string: <description-value>},
// tags: {string: <comma separated strings} // optional
// },
// <id>: {
// title: {string: <title-value>},
// description: {string: <description-value>},
// tags: {string: <comma separated strings} // optional
// }
// }
for (let [id, value] of Object.entries(json)) {
let body = {
title: value.title.string,
description: value.description.string,
status: 2 // set status to published
};
if (value.hasOwnProperty('tags')) {
let tags = value.tags.string.split(',');
let validTags = tags.filter(tag => tag.length < 33);
if (validTags.length !== tags.length) {
process.stdout.write(`Warning: tags too long in ${id} for ${locale}\n`);
}
body.tags = validTags;
}
let status = await FD.updateArticleTranslation(id, freshdeskLocale(locale), body);
if (status === -1) {
process.exitCode = 1;
}
}
return 0;
};
/**
* Process Transifex resource corresponding to a Knowledge base folder on Freshdesk
* @param {object} folder Transifex resource json corresponding to a KB folder
* @param {string} locale locale to pull and submit to Freshdesk
* @return {Promise} [description]
*/
exports.localizeFolder = async (folder, locale) => {
getTranslation(TX_PROJECT, folder.slug, locale, {mode: 'default'})
.then(data => {
const json = JSON.parse(data);
serializeFolderSave(json, locale);
})
.catch((e) => {
process.stdout.write(`Error processing ${folder.slug}, ${locale}: ${e.message}\n`);
process.exitCode = 1; // not ok
});
};
/**
* Save Transifex resource corresponding to a Knowledge base folder locally for debugging
* @param {object} folder Transifex resource json corresponding to a KB folder
* @param {string} locale locale to pull and save
* @return {Promise} [description]
*/
exports.debugFolder = async (folder, locale) => {
mkdirp.sync('tmpDebug');
getTranslation(TX_PROJECT, folder.slug, locale, {mode: 'default'})
.then(data => {
const json = JSON.parse(data);
fsPromises.writeFile(
`tmpDebug/${folder.slug}_${locale}.json`,
JSON.stringify(json, null, 2)
);
})
.catch((e) => {
process.stdout.write(`Error processing ${folder.slug}, ${locale}: ${e.message}\n`);
process.exitCode = 1; // not ok
});
};
/**
* Process KEYVALUEJSON resources from scratch-help on transifex
* Category and Folder names are stored as plain json
* @param {object} resource Transifex resource json for either CategoryNames or FolderNames
* @param {string} locale locale to pull and submit to Freshdesk
* @return {Promise} [description]
*/
exports.localizeNames = async (resource, locale) => {
getTranslation(TX_PROJECT, resource.slug, locale, {mode: 'default'})
.then(data => {
const json = JSON.parse(data);
serializeNameSave(json, resource, locale);
})
.catch((e) => {
process.stdout.write(`Error saving ${resource.slug}, ${locale}: ${e.message}\n`);
process.exitCode = 1; // not ok
});
};
const BATCH_SIZE = 2;
/*
* save resource items in batches to reduce rate limiting errors
* @param {object} item Transifex resource json, used for 'slug'
* @param {array} languages Array of languages to save
* @param {function} saveFn Async function to use to save the item
* @return {Promise}
*/
exports.saveItem = async (item, languages, saveFn) => {
const saveLanguages = languages.filter(l => l !== 'en'); // exclude English from update
let batchedPromises = Promise.resolve(); // eslint-disable-line no-undef
for (let i = 0; i < saveLanguages.length; i += BATCH_SIZE) {
batchedPromises = batchedPromises
.then(() => Promise.all( // eslint-disable-line
saveLanguages.slice(i, i + BATCH_SIZE).map(l => saveFn(item, l))
))
.catch(err => {
process.stdout.write(`Error saving item:${err.message}\n${JSON.stringify(item, null, 2)}\n`);
process.exitCode = 1; // not ok
});
}
};

34
scripts/tx-pull-help-articles.js Executable file
View file

@ -0,0 +1,34 @@
#!/usr/bin/env node
/**
* @fileoverview
* Script to pull scratch-help translations from transifex and push to FreshDesk.
*/
const args = process.argv.slice(2);
const usage = `
Pull knowledge base articles from transifex and push to FreshDesk. Usage:
node tx-pull-help.js
NOTE:
FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with
access to the Knowledge Base.
TX_TOKEN environment variable needs to be set with a Transifex API token. See
the Localization page on the GUI wiki for information about setting up Transifex.
`;
// Fail immediately if the API tokens are not defined, or there any argument
if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
process.stdout.write(usage);
process.exit(1);
}
const {getInputs, saveItem, localizeFolder} = require('./help-utils.js');
getInputs()
.then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars
process.stdout.write('Processing articles pulled from Transifex\n');
return folders.map(item => saveItem(item, languages, localizeFolder));
})
.catch((e) => {
process.stdout.write(`Error: ${e.message}\n`);
process.exitCode = 1; // not ok
});

35
scripts/tx-pull-help-names.js Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env node
/**
* @fileoverview
* Script to pull scratch-help translations from transifex and push to FreshDesk.
*/
const args = process.argv.slice(2);
const usage = `
Pull knowledge base category and folder names from transifex and push to FreshDesk. Usage:
node tx-pull-help.js
NOTE:
FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with
access to the Knowledge Base.
TX_TOKEN environment variable needs to be set with a Transifex API token. See
the Localization page on the GUI wiki for information about setting up Transifex.
`;
// Fail immediately if the API tokens are not defined, or there any argument
if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
process.stdout.write(usage);
process.exit(1);
}
const {getInputs, saveItem, localizeNames} = require('./help-utils.js');
getInputs()
.then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars
process.stdout.write('Process Category and Folder Names pulled from Transifex\n');
return names.map(item => saveItem(item, languages, localizeNames));
})
.catch((e) => {
process.stdout.write(`Error: ${e.message}\n`);
process.exitCode = 1; // not ok
});

42
scripts/tx-pull-locale-articles.js Executable file
View file

@ -0,0 +1,42 @@
#!/usr/bin/env node
/**
* @fileoverview
* Script to pull scratch-help translations from transifex and push to FreshDesk.
*/
const args = process.argv.slice(2);
const usage = `
Pull knowledge base articles from transifexfor debugging translation errors. Usage:
node tx-pull-locale-articles.js -d locale-code
NOTE:
FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with
access to the Knowledge Base.
TX_TOKEN environment variable needs to be set with a Transifex API token. See
the Localization page on the GUI wiki for information about setting up Transifex.
`;
// Fail immediately if the API tokens are not defined, or missing argument
if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length === 0) {
process.stdout.write(usage);
process.exit(1);
}
const {getInputs, saveItem, localizeFolder, debugFolder} = require('./help-utils.js');
let locale = args[0];
let debug = false;
if (locale === '-d') {
debug = true;
locale = args[1];
}
const saveFn = debug ? debugFolder : localizeFolder;
getInputs()
.then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars
process.stdout.write('Processing articles pulled from Transifex\n');
return folders.map(item => saveItem(item, [locale], saveFn));
})
.catch((e) => {
process.stdout.write(`Error: ${e.message}\n`);
process.exitCode = 1; // not ok
});

146
scripts/tx-push-help.js Executable file
View file

@ -0,0 +1,146 @@
#!/usr/bin/env babel-node
/**
* @fileoverview
* Script get Knowledge base articles from Freshdesk and push them to transifex.
*/
const args = process.argv.slice(2);
const usage = `
Pull knowledge base articles from Freshdesk and push to scratch-help project on transifex. Usage:
node tx-push-help.js
NOTE:
FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with
access to the Knowledge Base.
TX_TOKEN environment variable needs to be set with a Transifex API token. See
the Localization page on the GUI wiki for information about setting up Transifex.
`;
// Fail immediately if the API tokens are not defined, or there any argument
if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
process.stdout.write(usage);
process.exit(1);
}
import transifex from 'transifex';
import FreshdeskApi from './freshdesk-api.js';
const FD = new FreshdeskApi('https://mitscratch.freshdesk.com', process.env.FRESHDESK_TOKEN);
const TX_PROJECT = 'scratch-help';
const TX = new transifex({
project_slug: TX_PROJECT,
credential: 'api:' + process.env.TX_TOKEN
});
const categoryNames = {};
const folderNames = {};
/**
* Generate a transifex id from the name and id field of an objects. Remove spaces and '/'
* from the name and append '.<id>' Transifex ids (slugs) have a max length of 50. Use at most
* 30 characters of the name to allow for Freshdesk id, and a suffix like '_json'
* @param {object} item data from Freshdesk that includes the name and id of a category or folder
* @return {string} generated transifex id
*/
const makeTxId = item => {
return `${item.name.replace(/[ /]/g, '').slice(0, 30)}_${item.id}`;
};
const txPushResource = (name, articles, type) => {
const resourceData = {
slug: name,
name: name,
priority: 0, // default to normal priority
i18n_type: type,
content: '{}'
};
TX.resourceCreateMethod(TX_PROJECT, resourceData, (err) => {
// ignore already created error report others
if (err && err.response.statusCode !== 400) {
process.stdout.write(`Transifex Error: ${err.message}\n`);
process.stdout.write(
`Transifex Error ${err.response.statusCode.toString()}: ${err.response.body}\n`);
process.exitCode = 1;
return;
}
// update Transifex with English source
TX.uploadSourceLanguageMethod(TX_PROJECT, name,
{content: JSON.stringify(articles, null, 2)}, (err1) => {
if (err1) {
process.stdout.write(`Transifex Error:${err1.name}, ${err1.message}\n`);
process.stdout.write(`Transifex Error:${err1.toString()}\n`);
process.exitCode = 1;
}
});
});
};
/**
* get a flattened list of folders
* @param {category} categories array of categories the folders belong to
* @return {Promise} flattened list of folders
*/
const getFolders = async (categories) => {
let categoryFolders = await Promise.all( // eslint-disable-line no-undef
categories.map(category => FD.listFolders(category))
);
return [].concat(...categoryFolders);
};
const PUBLISHED = 2; // in Freshdesk, draft status = 1, and published = 2
const saveArticles = (folder) => {
FD.listArticles(folder)
.then(json => {
let txArticles = json.reduce((strings, current) => {
if (current.status === PUBLISHED) {
strings[`${current.id}`] = {
title: {
string: current.title
},
description: {
string: current.description
}
};
if (current.tags.length > 0) {
strings[`${current.id}`].tags = {string: current.tags.toString()};
}
}
return strings;
}, {});
process.stdout.write(`Push ${folder.name} articles to Transifex\n`);
txPushResource(`${makeTxId(folder)}_json`, txArticles, 'STRUCTURED_JSON');
});
};
const getArticles = async (folders) => {
return Promise.all(folders.map(folder => saveArticles(folder))); // eslint-disable-line no-undef
};
const syncSources = async () => {
let status = 0;
status = await FD.listCategories()
.then(json => {
// save category names for translation
for (let cat of json.values()) {
categoryNames[`${makeTxId(cat)}`] = cat.name;
}
return json;
})
.then(getFolders)
.then(data => {
data.forEach(item => {
folderNames[`${makeTxId(item)}`] = item.name;
});
process.stdout.write('Push category and folder names to Transifex\n');
txPushResource('categoryNames_json', categoryNames, 'KEYVALUEJSON');
txPushResource('folderNames_json', folderNames, 'KEYVALUEJSON');
return data;
})
.then(getArticles)
.catch((e) => {
process.stdout.write(`Error:${e.message}\n`);
return 1;
});
process.exitCode = status;
};
syncSources();