From 30b1975e9408d6f7db0c2f63bc08ea7fcf1c1055 Mon Sep 17 00:00:00 2001 From: Chris Garrity Date: Wed, 6 May 2020 14:44:46 -0400 Subject: [PATCH] Scripts to pull translations and update Freshdesk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixed a bug in the push script (tags were not saved as correct structured json, so they weren’t getting put into transifex) * extended FreshDesk API with functions to update knowledge base. Functions automatically try to create the item if it isn’t found for updating * tx-pull-help-names: pulls category and folder name translations from transifex and updates them in Freshdesk (Note, since we don’t send people into the knowledge base, these aren’t really public, but the actual folders and categories have to exist to be able to save articles) * tx-pull-help-articles: pull article translations from Transifex and update the Freshdesk KB. * help-utils: utility functions for the tx-pull-help-* scripts. Handles limiting the number of things happening in parallel - currently two languages may be processed at the same time. --- scripts/freshdesk-api.js | 104 ++++++++++++++++- scripts/help-utils.js | 189 +++++++++++++++++++++++++++++++ scripts/tx-pull-help-articles.js | 34 ++++++ scripts/tx-pull-help-names.js | 35 ++++++ scripts/tx-push-help.js | 2 +- 5 files changed, 362 insertions(+), 2 deletions(-) create mode 100644 scripts/help-utils.js create mode 100755 scripts/tx-pull-help-articles.js create mode 100755 scripts/tx-pull-help-names.js diff --git a/scripts/freshdesk-api.js b/scripts/freshdesk-api.js index dffebfc6..e8f067a5 100644 --- a/scripts/freshdesk-api.js +++ b/scripts/freshdesk-api.js @@ -10,6 +10,7 @@ class FreshdeskApi { 'Content-Type': 'application/json', 'Authorization': this._auth }; + this.rateLimited = false; } /** @@ -24,7 +25,9 @@ class FreshdeskApi { } throw new Error(`response not json: ${res.headers.get('content-type')}`); } - throw new Error(`response: ${res.statusText}`); + let err = new Error(`response ${res.statusText}`); + err.code = res.status; + throw err; } listCategories () { @@ -48,6 +51,105 @@ class FreshdeskApi { .then(this.checkStatus) .then(res => res.json()); } + + updateCategoryTranslation (id, locale, body) { + if (this.rateLimited) { + process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`); + return -1; + } + return fetch( + `${this.baseUrl}/api/v2/solutions/categories/${id}/${locale}`, + { + method: 'put', + body: JSON.stringify(body), + headers: this.defaultHeaders + }) + .then(this.checkStatus) + .then(res => res.json()) + .catch((err) => { + if (err.code === 404) { + // not found, try create instead + return fetch( + `${this.baseUrl}/api/v2/solutions/categories/${id}/${locale}`, + { + method: 'post', + body: JSON.stringify(body), + headers: this.defaultHeaders + }) + .then(this.checkStatus) + .then(res => res.json()); + } + // re-raise the error otherwise + throw err; + }); + } + + updateFolderTranslation (id, locale, body) { + if (this.rateLimited) { + process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`); + return -1; + } + return fetch( + `${this.baseUrl}/api/v2/solutions/folders/${id}/${locale}`, + { + method: 'put', + body: JSON.stringify(body), + headers: this.defaultHeaders + }) + .then(this.checkStatus) + .then(res => res.json()) + .catch((err) => { + if (err.code === 404) { + // not found, try create instead + return fetch( + `${this.baseUrl}/api/v2/solutions/folders/${id}/${locale}`, + { + method: 'post', + body: JSON.stringify(body), + headers: this.defaultHeaders + }) + .then(this.checkStatus) + .then(res => res.json()); + } + // re-raise the error otherwise + throw err; + }); + } + + updateArticleTranslation (id, locale, body) { + if (this.rateLimited) { + process.stdout.write(`Rate limited, skipping id: ${id} for ${locale}\n`); + return -1; + } + return fetch( + `${this.baseUrl}/api/v2/solutions/articles/${id}/${locale}`, + { + method: 'put', + body: JSON.stringify(body), + headers: this.defaultHeaders + }) + .then(this.checkStatus) + .then(res => res.json()) + .catch((err) => { + if (err.code === 404) { + // not found, try create instead + return fetch( + `${this.baseUrl}/api/v2/solutions/articles/${id}/${locale}`, + { + method: 'post', + body: JSON.stringify(body), + headers: this.defaultHeaders + }) + .then(this.checkStatus) + .then(res => res.json()); + } + if (err.code === 429) { + this.rateLimited = true; + } + // re-raise the error otherwise + throw err; + }); + } } module.exports = FreshdeskApi; diff --git a/scripts/help-utils.js b/scripts/help-utils.js new file mode 100644 index 00000000..cdb72004 --- /dev/null +++ b/scripts/help-utils.js @@ -0,0 +1,189 @@ +#!/usr/bin/env node + +/** + * @fileoverview + * Helper functions for syncing Freshdesk knowledge base articles with Transifex + */ + +const transifex = require('transifex'); +const FreshdeskApi = require('./freshdesk-api.js'); +const util = require('util'); +// const fs = require('fs'); +// const mkdirp = require('mkdirp'); + +const FD = new FreshdeskApi('https://mitscratch.freshdesk.com', process.env.FRESHDESK_TOKEN); +const TX_PROJECT = 'scratch-help'; +const TX = new transifex({ + project_slug: TX_PROJECT, + credential: 'api:' + process.env.TX_TOKEN +}); + +const freshdeskLocale = locale => { + // map between Transifex locale and Freshdesk. Two letter codes are usually fine + const localeMap = { + 'es_419': 'es-LA', + 'ja': 'ja-JP', + 'ja-Hira': 'ja-JP', + 'lv': 'lv-LV', + 'nb': 'nb-NO', + 'nn': 'nb-NO', + 'pt': 'pt-PT', + 'pt_BR': 'pt-BR', + 'ru': 'ru-RU', + 'sv': 'sv-SE', + 'zh_CN': 'zh-CN', + 'zh_TW': 'zh-TW' + }; + return localeMap[locale] || locale; +}; + +// Promisify Transifex async/callbacks to make them easier to use` +const getResources = util.promisify(TX.resourcesSetMethod.bind(TX)); +const getResourceInfo = util.promisify(TX.resourcesInstanceMethods.bind(TX)); +const getTranslation = util.promisify(TX.translationInstanceMethod.bind(TX)); + +/** + * Pull metadata from Transifex for the scratch-help project + * @return {Promise} results array containing: + * languages: array of supported languages + * folders: array of tx resources corrsponding to Freshdesk folders + * names: array of tx resources corresponding to the Freshdesk metadata + */ +exports.getInputs = async () => { + const resources = await getResources(TX_PROJECT); + // there are three types of resources differentiated by the file type + const folders = resources.filter(resource => resource.i18n_type === 'STRUCTURED_JSON'); + const names = resources.filter(resource => resource.i18n_type === 'KEYVALUEJSON'); + // ignore the yaml type because it's not possible to update via API + + // Lookup available languages by getting metadata for a resource, they all have the + // same set of languages, so it doesn't matter which one you get + const resourceInfo = await getResourceInfo(TX_PROJECT, resources[0].slug, true); + const languages = resourceInfo.available_languages.map(l => l.code); + + return Promise.all([languages, folders, names]); // eslint-disable-line no-undef +}; + +/** + * internal function to serialize saving category and folder name translations to avoid Freshdesk rate limit + * @param {[type]} json [description] + * @param {[type]} resource [description] + * @param {[type]} locale [description] + * @return {Promise} [description] + */ +const serializeNameSave = async (json, resource, locale) => { + for (let [key, value] of Object.entries(json)) { + // key is of the form _ + const words = key.split('_'); + const id = words[words.length - 1]; + let status = 0; + if (resource.name === 'categoryNames_json') { + status = await FD.updateCategoryTranslation(id, freshdeskLocale(locale), {name: value}); + } + if (resource.name === 'folderNames_json') { + status = await FD.updateFolderTranslation(id, freshdeskLocale(locale), {name: value}); + } + if (status === -1) { + process.exitCode = 1; + } + } +}; + +/** + * Internal function serialize Freshdesk requests to avoid getting rate limited + * @param {object} json object with keys corresponding to article ids + * @param {string} locale language code + * @return {Promise} [description] + */ +const serializeFolderSave = async (json, locale) => { + // json is a map of articles: + // { + // : { + // title: {string: }, + // description: {string: }, + // tags: {string: : { + // title: {string: }, + // description: {string: }, + // tags: {string: { + getTranslation(TX_PROJECT, folder.slug, locale, {mode: 'default'}) + .then(data => { + const json = JSON.parse(data); + + serializeFolderSave(json, locale); + }) + .catch((e) => { + process.stdout.write(`Error processing ${folder.slug}, ${locale}: ${e.message}\n`); + process.exitCode = 1; // not ok + }); +}; + +/** + * Process KEYVALUEJSON resources from scratch-help on transifex + * Category and Folder names are stored as plain json + * @param {object} resource Transifex resource json for either CategoryNames or FolderNames + * @param {string} locale locale to pull and submit to Freshdesk + * @return {Promise} [description] + */ +exports.localizeNames = async (resource, locale) => { + getTranslation(TX_PROJECT, resource.slug, locale, {mode: 'default'}) + .then(data => { + const json = JSON.parse(data); + serializeNameSave(json, resource, locale); + }) + .catch((e) => { + process.stdout.write(`Error saving ${resource.slug}, ${locale}: ${e.message}\n`); + process.exitCode = 1; // not ok + }); +}; + + +const BATCH_SIZE = 2; +/* + * save resource items in batches to reduce rate limiting errors + * @param {object} item Transifex resource json, used for 'slug' + * @param {array} languages Array of languages to save + * @param {function} saveFn Async function to use to save the item + * @return {Promise} + */ +exports.saveItem = async (item, languages, saveFn) => { + const saveLanguages = languages.filter(l => l !== 'en'); // exclude English from update + let batchedPromises = Promise.resolve(); // eslint-disable-line no-undef + for (let i = 0; i < saveLanguages.length; i += BATCH_SIZE) { + batchedPromises = batchedPromises + .then(() => Promise.all( // eslint-disable-line + saveLanguages.slice(i, i + BATCH_SIZE).map(l => saveFn(item, l)) + )) + .catch(err => { + process.stdout.write(`Error saving item:${err.message}\n${JSON.stringify(item, null, 2)}\n`); + process.exitCode = 1; // not ok + }); + } +}; diff --git a/scripts/tx-pull-help-articles.js b/scripts/tx-pull-help-articles.js new file mode 100755 index 00000000..8f6985e2 --- /dev/null +++ b/scripts/tx-pull-help-articles.js @@ -0,0 +1,34 @@ +#!/usr/bin/env node + +/** + * @fileoverview + * Script to pull scratch-help translations from transifex and push to FreshDesk. + */ + +const args = process.argv.slice(2); +const usage = ` + Pull knowledge base articles from transifex and push to FreshDesk. Usage: + node tx-pull-help.js + NOTE: + FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with + access to the Knowledge Base. + TX_TOKEN environment variable needs to be set with a Transifex API token. See + the Localization page on the GUI wiki for information about setting up Transifex. + `; +// Fail immediately if the API tokens are not defined, or there any argument +if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) { + process.stdout.write(usage); + process.exit(1); +} + +const {getInputs, saveItem, localizeFolder} = require('./help-utils.js'); + +getInputs() + .then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars + process.stdout.write('Processing articles pulled from Transifex\n'); + return folders.map(item => saveItem(item, languages, localizeFolder)); + }) + .catch((e) => { + process.stdout.write(`Error: ${e.message}\n`); + process.exitCode = 1; // not ok + }); diff --git a/scripts/tx-pull-help-names.js b/scripts/tx-pull-help-names.js new file mode 100755 index 00000000..13094367 --- /dev/null +++ b/scripts/tx-pull-help-names.js @@ -0,0 +1,35 @@ +#!/usr/bin/env node + +/** + * @fileoverview + * Script to pull scratch-help translations from transifex and push to FreshDesk. + */ + +const args = process.argv.slice(2); + +const usage = ` + Pull knowledge base category and folder names from transifex and push to FreshDesk. Usage: + node tx-pull-help.js + NOTE: + FRESHDESK_TOKEN environment variable needs to be set to a FreshDesk API key with + access to the Knowledge Base. + TX_TOKEN environment variable needs to be set with a Transifex API token. See + the Localization page on the GUI wiki for information about setting up Transifex. + `; +// Fail immediately if the API tokens are not defined, or there any argument +if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) { + process.stdout.write(usage); + process.exit(1); +} + +const {getInputs, saveItem, localizeNames} = require('./help-utils.js'); + +getInputs() + .then(([languages, folders, names]) => { // eslint-disable-line no-unused-vars + process.stdout.write('Process Category and Folder Names pulled from Transifex\n'); + return names.map(item => saveItem(item, languages, localizeNames)); + }) + .catch((e) => { + process.stdout.write(`Error: ${e.message}\n`); + process.exitCode = 1; // not ok + }); diff --git a/scripts/tx-push-help.js b/scripts/tx-push-help.js index c93798eb..a9d60d37 100755 --- a/scripts/tx-push-help.js +++ b/scripts/tx-push-help.js @@ -102,7 +102,7 @@ const saveArticles = (folder) => { } }; if (current.tags.length > 0) { - strings[`${current.id}`].tags = current.tags.toString(); + strings[`${current.id}`].tags = {string: current.tags.toString()}; } } return strings;