Merge pull request #18 from cwillisf/telemetry

Implement telemetry client

package-lock.json

@@ -2930,6 +2930,19 @@
         "typedarray": "^0.0.6"
       }
     },
+    "conf": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/conf/-/conf-2.1.0.tgz",
+      "integrity": "sha512-IcWtHiBjeNtyCG+XK/v9Pz8Q4+nsyvO60Zabn6SsHTR2TMaLN6os/jrUtuQnATb12RI82RHKt+PVEXTsH6XMXQ==",
+      "dev": true,
+      "requires": {
+        "dot-prop": "^4.1.0",
+        "env-paths": "^1.0.0",
+        "make-dir": "^1.0.0",
+        "pkg-up": "^2.0.0",
+        "write-file-atomic": "^2.3.0"
+      }
+    },
     "configstore": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/configstore/-/configstore-3.1.2.tgz",
@@ -3719,6 +3732,12 @@
         }
       }
     },
+    "dom-walk": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.1.tgz",
+      "integrity": "sha1-ZyIm3HTI95mtNTB9+TaroRrNYBg=",
+      "dev": true
+    },
     "domain-browser": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
@@ -3932,6 +3951,15 @@
         "mime": "^2.4.0"
       }
     },
+    "electron-store": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/electron-store/-/electron-store-2.0.0.tgz",
+      "integrity": "sha512-1WCFYHsYvZBqDsoaS0Relnz0rd81ZkBAI0Fgx7Nq2UWU77rSNs1qxm4S6uH7TCZ0bV3LQpJFk7id/is/ZgoOPA==",
+      "dev": true,
+      "requires": {
+        "conf": "^2.0.0"
+      }
+    },
     "electron-to-chromium": {
       "version": "1.3.79",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.79.tgz",
@@ -5029,6 +5057,15 @@
         }
       }
     },
+    "for-each": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
+      "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==",
+      "dev": true,
+      "requires": {
+        "is-callable": "^1.1.3"
+      }
+    },
     "for-in": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
@@ -5744,6 +5781,24 @@
         }
       }
     },
+    "global": {
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/global/-/global-4.3.2.tgz",
+      "integrity": "sha1-52mJJopsdMOJCLEwWxD8DjlOnQ8=",
+      "dev": true,
+      "requires": {
+        "min-document": "^2.19.0",
+        "process": "~0.5.1"
+      },
+      "dependencies": {
+        "process": {
+          "version": "0.5.2",
+          "resolved": "https://registry.npmjs.org/process/-/process-0.5.2.tgz",
+          "integrity": "sha1-FjjYqONML0QKkduVq5rrZ3/Bhc8=",
+          "dev": true
+        }
+      }
+    },
     "global-dirs": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-0.1.1.tgz",
@@ -6608,6 +6663,12 @@
         "number-is-nan": "^1.0.0"
       }
     },
+    "is-function": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.1.tgz",
+      "integrity": "sha1-Es+5i2W1fdPRk6MSH19uL0N2ArU=",
+      "dev": true
+    },
     "is-glob": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
@@ -7301,6 +7362,15 @@
       "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==",
       "dev": true
     },
+    "min-document": {
+      "version": "2.19.0",
+      "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz",
+      "integrity": "sha1-e9KC4/WELtKVu3SM3Z8f+iyCRoU=",
+      "dev": true,
+      "requires": {
+        "dom-walk": "^0.1.0"
+      }
+    },
     "mini-css-extract-plugin": {
       "version": "0.4.5",
       "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-0.4.5.tgz",
@@ -7473,6 +7543,16 @@
       "integrity": "sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA==",
       "dev": true
     },
+    "nets": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/nets/-/nets-3.2.0.tgz",
+      "integrity": "sha1-1RH7q3rxHaAT8huX7pF0fTOFLTg=",
+      "dev": true,
+      "requires": {
+        "request": "^2.65.0",
+        "xhr": "^2.1.0"
+      }
+    },
     "nice-try": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
@@ -8002,6 +8082,16 @@
         }
       }
     },
+    "parse-headers": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.1.tgz",
+      "integrity": "sha1-aug6eqJanZtwCswoaYzR8e1+lTY=",
+      "dev": true,
+      "requires": {
+        "for-each": "^0.3.2",
+        "trim": "0.0.1"
+      }
+    },
     "parse-json": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz",
@@ -8135,6 +8225,15 @@
         "find-up": "^2.1.0"
       }
     },
+    "pkg-up": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-2.0.0.tgz",
+      "integrity": "sha1-yBmscoBZpGHKscOImivjxJoATX8=",
+      "dev": true,
+      "requires": {
+        "find-up": "^2.1.0"
+      }
+    },
     "plist": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.1.tgz",
@@ -12323,6 +12422,12 @@
         "punycode": "^1.4.1"
       }
     },
+    "trim": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.1.tgz",
+      "integrity": "sha1-WFhUf2spB1fulczMZm+1AITEYN0=",
+      "dev": true
+    },
     "trim-newlines": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
@@ -13304,6 +13409,18 @@
       "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=",
       "dev": true
     },
+    "xhr": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.5.0.tgz",
+      "integrity": "sha512-4nlO/14t3BNUZRXIXfXe+3N6w3s1KoxcJUUURctd64BLRe67E4gRwp4PjywtDY72fXpZ1y6Ch0VZQRY/gMPzzQ==",
+      "dev": true,
+      "requires": {
+        "global": "~4.3.0",
+        "is-function": "^1.0.1",
+        "parse-headers": "^2.0.0",
+        "xtend": "^4.0.0"
+      }
+    },
     "xmlbuilder": {
       "version": "9.0.7",
       "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",

package.json

@@ -36,6 +36,7 @@
     "electron": "^3.0.10",
     "electron-builder": "^20.38.2",
     "electron-devtools-installer": "^2.2.4",
+    "electron-store": "^2.0.0",
     "electron-webpack": "^2.6.1",
     "eslint": "^5.9.0",
     "eslint-config-scratch": "^5.0.0",
@@ -43,6 +44,7 @@
     "eslint-plugin-react": "^7.5.1",
     "intl": "1.2.5",
     "mkdirp": "^0.5.1",
+    "nets": "^3.2.0",
     "react": "16.2.0",
     "react-dom": "16.2.0",
     "react-intl": "2.4.0",
@@ -52,6 +54,7 @@
     "scratch-gui": "0.1.0-prerelease.20180927141400",
     "source-map-loader": "^0.2.4",
     "uglifyjs-webpack-plugin": "^1.2.5",
+    "uuid": "^3.3.2",
     "webpack": "^4.27.0"
   },
   "resolutions": {

src/main/ScratchDesktopTelemetry.js

@@ -0,0 +1,83 @@
+import {app, ipcMain} from 'electron';
+
+import TelemetryClient from './telemetry/TelemetryClient';
+
+const EVENT_TEMPLATE = {
+    version: '3.0.0',
+    projectName: '',
+    language: '',
+    scriptCount: -1,
+    spriteCount: -1,
+    variablesCount: -1,
+    blocksCount: -1,
+    costumesCount: -1,
+    listsCount: -1,
+    soundsCount: -1
+};
+
+const APP_ID = 'scratch-desktop';
+const APP_VERSION = app.getVersion();
+const APP_INFO = Object.freeze({
+    projectName: `${APP_ID} ${APP_VERSION}`
+});
+
+class ScratchDesktopTelemetry {
+    constructor () {
+        this._telemetryClient = new TelemetryClient();
+    }
+
+    get didOptIn () {
+        return this._telemetryClient.didOptIn;
+    }
+    set didOptIn (value) {
+        this._telemetryClient.didOptIn = value;
+    }
+
+    appWasOpened () {
+        this._telemetryClient.addEvent('app::open', {...EVENT_TEMPLATE, ...APP_INFO});
+    }
+
+    appWillClose () {
+        this._telemetryClient.addEvent('app::close', {...EVENT_TEMPLATE, ...APP_INFO});
+    }
+
+    projectDidLoad (metadata = {}) {
+        this._telemetryClient.addEvent('project::load', {...EVENT_TEMPLATE, ...metadata});
+    }
+
+    projectDidSave (metadata = {}) {
+        this._telemetryClient.addEvent('project::save', {...EVENT_TEMPLATE, ...metadata});
+    }
+
+    projectWasCreated (metadata = {}) {
+        this._telemetryClient.addEvent('project::create', {...EVENT_TEMPLATE, ...metadata});
+    }
+
+    projectWasUploaded (metadata = {}) {
+        this._telemetryClient.addEvent('project::upload', {...EVENT_TEMPLATE, ...metadata});
+    }
+}
+
+// make a singleton so it's easy to share across both Electron processes
+const scratchDesktopTelemetrySingleton = new ScratchDesktopTelemetry();
+
+ipcMain.on('getTelemetryDidOptIn', event => {
+    event.returnValue = scratchDesktopTelemetrySingleton.didOptIn;
+});
+ipcMain.on('setTelemetryDidOptIn', (event, arg) => {
+    scratchDesktopTelemetrySingleton.didOptIn = arg;
+});
+ipcMain.on('projectDidLoad', (event, arg) => {
+    scratchDesktopTelemetrySingleton.projectDidLoad(arg);
+});
+ipcMain.on('projectDidSave', (event, arg) => {
+    scratchDesktopTelemetrySingleton.projectDidSave(arg);
+});
+ipcMain.on('projectWasCreated', (event, arg) => {
+    scratchDesktopTelemetrySingleton.projectWasCreated(arg);
+});
+ipcMain.on('projectWasUploaded', (event, arg) => {
+    scratchDesktopTelemetrySingleton.projectWasUploaded(arg);
+});
+
+export default scratchDesktopTelemetrySingleton;

src/main/index.js

@@ -1,6 +1,10 @@
 import {BrowserWindow, app, dialog} from 'electron';
 import * as path from 'path';
 import {format as formatUrl} from 'url';
+import telemetry from './ScratchDesktopTelemetry';
+
+telemetry.appWasOpened();
+
 
 // const defaultSize = {width: 1096, height: 715}; // minimum
 const defaultSize = {width: 1280, height: 800}; // good for MAS screenshots
@@ -71,6 +75,10 @@ app.on('window-all-closed', () => {
     app.quit();
 });
 
+app.on('will-quit', () => {
+    telemetry.appWillClose();
+});
+
 // global reference to mainWindow (necessary to prevent window from being garbage collected)
 let _mainWindow;
 

src/main/telemetry/TelemetryClient.js

@@ -0,0 +1,250 @@
+import ElectronStore from 'electron-store';
+import nets from 'nets';
+import uuidv1 from 'uuid/v1'; // semi-persistent client ID
+import uuidv4 from 'uuid/v4'; // random ID
+
+/**
+ * Basic telemetry event data. These fields are filled automatically by the `addEvent` call.
+ * @typedef {object} BasicTelemetryEvent
+ * @property {string} clientID - a UUID for this client
+ * @property {string} id - a UUID for this event/packet
+ * @property {string} name - the name of this event (taken from `addEvent`'s `eventName` parameter)
+ * @property {int} timestamp - a Unix epoch timestamp for this event
+ * @property {int} userTimezone - the difference in minutes between UTC and local time
+ */
+
+/**
+  * Default telemetry service URLs
+  */
+const TelemetryServerURL = Object.freeze({
+    staging: 'http://scratch-telemetry-s.us-east-1.elasticbeanstalk.com/',
+    production: 'https://telemetry.scratch.mit.edu/'
+});
+
+/**
+ * Default interval, in seconds, between delivery attempts
+ */
+const DefaultDeliveryInterval = 60;
+
+/**
+ * Default interval, in seconds, between connectivity checks
+ */
+const DefaultNetworkCheckInterval = 300;
+
+/**
+ * Client interface for the Scratch telemetry service.
+ *
+ * This class supports delivering generic telemetry events and is designed to be used by any application or service
+ * in the Scratch family.
+ */
+class TelemetryClient {
+    /**
+     * Construct and initialize a TelemetryClient instance, optionally overriding configuration defaults. Delivery
+     * intervals will begin immediately; if the user has not opted in events will be dropped each interval.
+     *
+     * @param {object} [options] - optional configuration settings for this client
+     * @property {string} [storeName] - optional name for persistent config/queue storage (default: 'telemetry')
+     * @property {string} [clientId] - optional UUID for this client (default: automatically determine a UUID)
+     * @property {string} [url] - optional telemetry service endpoint URL (default: automatically choose a server)
+     * @property {boolean} [didOptIn] - optional flag for whether the user opted into telemetry service (default: false)
+     * @property {int} [deliveryInterval] - optional number of seconds between delivery attempts (default: 60)
+     * @property {int} [networkInterval] - optional number of seconds between connectivity checks (default: 300)
+     * @property {int} [queueLimit] - optional limit on the number of queued events (default: 100)
+     * @property {int} [attemptLimit] - optional limit on the number of delivery attempts for each event (default: 3)
+     */
+    constructor (options = null) {
+        options = options || {};
+
+        /**
+         * Persistent storage for the client ID, opt in flag, and packet queue.
+         */
+        this._store = new ElectronStore({
+            name: options.storeName || 'telemetry'
+        });
+        console.log(`Telemetry configuration storage path: ${this._store.path}`);
+
+        if (options.hasOwnProperty('clientID')) {
+            this.clientID = options.clientID;
+        } else if (!this._store.has('clientID')) {
+            this.clientID = uuidv1();
+        }
+
+        if (options.hasOwnProperty('optIn')) {
+            this.didOptIn = options.didOptIn;
+        }
+
+        /**
+         * Queue for outgoing event packets
+         */
+        this._packetQueue = this._store.get('packetQueue', []);
+
+        /**
+         * Server URL
+         */
+        this._serverURL = options.url || TelemetryServerURL.staging;
+
+        /**
+         * Can we currently reach the telemetry service?
+         */
+        this._networkIsOnline = false;
+
+        /**
+         * Try to deliver telemetry packets at this interval
+         */
+        this._deliveryInterval = (options.deliveryInterval > 0) ? options.deliveryInterval : DefaultDeliveryInterval;
+
+        /**
+         * Check for connectivity at this interval
+         */
+        this._networkCheckInterval =
+            (options.networkCheckInterval > 0) ? options.networkCheckInterval : DefaultNetworkCheckInterval;
+
+        /**
+         * Bind event handlers
+         */
+        this._attemptDelivery = this._attemptDelivery.bind(this);
+        this._updateNetworkStatus = this._updateNetworkStatus.bind(this);
+
+        /**
+         * Begin monitoring network status
+         */
+        this._networkTimer = setInterval(this._updateNetworkStatus, this._networkCheckInterval * 1000);
+        setTimeout(this._updateNetworkStatus, 0);
+
+        /**
+         * Begin the delivery interval
+         */
+        this._deliveryTimer = setInterval(this._attemptDelivery, this._deliveryInterval * 1000);
+    }
+
+    /**
+     * Stop this client. Do not use this object after disposal.
+     */
+    dispose () {
+        if (this._deliveryInterval !== null) {
+            clearInterval(this._deliveryTimer);
+            this._deliveryInterval = null;
+        }
+    }
+
+    /**
+     * Has the user explicitly opted into this service?
+     * @type {boolean}
+     */
+    get didOptIn () {
+        // don't supply a default here: we want to track "opt out" separately from "undecided"
+        return this._store.get('optIn');
+    }
+    set didOptIn (value) {
+        this._store.set('optIn', !!value);
+    }
+
+    /**
+     * Semi-persistent unique ID for this client
+     * @type {string}
+     */
+    get clientID () {
+        return this._store.get('clientID');
+    }
+    set clientID (value) {
+        this._store.set('clientID', value);
+    }
+
+    /**
+     * Save the packet queue to the config store.
+     * Call this any time the queue is modified.
+     */
+    saveQueue () {
+        this._store.set('packetQueue', this._packetQueue);
+    }
+
+    /**
+     * Add an event to the telemetry system. If the user has opted into the telemetry service, this event will be
+     * delivered to the telemetry service when possible. Otherwise the event will be ignored.
+     *
+     * @see {@link BasicTelemetryEvent} for the list of fields which are filled automatically by this method.
+     *
+     * @param {string} eventName - the name of this telemetry event, such as 'app::open'.
+     * @param {object} additionalFields - optional event fields to add or override before sending the event.
+     */
+    addEvent (eventName, additionalFields = null) {
+        const packetId = uuidv4();
+        const now = new Date();
+
+        const packet = Object.assign({
+            clientID: this.clientID,
+            id: packetId,
+            name: eventName,
+            timestamp: now.getTime(),
+            userTimezone: now.getTimezoneOffset()
+        }, additionalFields);
+        const packetInfo = {
+            attempts: 0,
+            packet
+        };
+        this._packetQueue.push(packetInfo);
+        this.saveQueue();
+    }
+
+    /**
+     * Attempt to deliver events to the telemetry service. If telemetry is disabled, this will do nothing.
+     */
+    _attemptDelivery () {
+        if (this._busy) {
+            return;
+        }
+
+        /**
+         * Attempt to deliver one event then asynchronously recurse, reenqueueing the event if delivery fails and the
+         * event has not yet reached its retry limit. Sets `this._busy` before doing anything else and clears it once
+         * the queue is empty or `this.didOptIn` is cleared.
+         */
+        const stepDelivery = () => {
+            this._busy = true;
+            if (!this.didOptIn || !this._networkIsOnline || this._packetQueue.length < 1) {
+                this._busy = false;
+                return;
+            }
+            // don't saveQueue() here:
+            // - if the app exits or crashes before the network request finishes, we'll lose the packet
+            // - if the request finishes, we'll save at that time (see below)
+            const packetInfo = this._packetQueue.shift();
+            ++packetInfo.attempts;
+            const packet = packetInfo.packet;
+            nets({
+                body: JSON.stringify(packet),
+                headers: {'Content-Type': 'application/json'},
+                method: 'POST',
+                url: this._serverURL
+            }, (err, res) => {
+                // TODO: check if the failure is because there's no Internet connection and if so refund the attempt
+                const packetFailed = err || (res.statusCode !== 200);
+                if (packetFailed) {
+                    if (packetInfo.attempts < this._attemptsLimit) {
+                        this._packetQueue.push(packetInfo);
+                    } else {
+                        console.warn('Dropping packet which exceeded retry limit', packet);
+                    }
+                }
+                this.saveQueue();
+                stepDelivery();
+            });
+        };
+
+        stepDelivery();
+    }
+
+    /**
+     * Check if the telemetry service is available
+     */
+    _updateNetworkStatus () {
+        nets({
+            method: 'GET',
+            url: this._serverURL
+        }, (err, res) => {
+            this._networkIsOnline = !err && (res.statusCode === 200);
+        });
+    }
+}
+
+export default TelemetryClient;

src/renderer/index.js

@@ -1,3 +1,4 @@
+import {ipcRenderer} from 'electron';
 import React from 'react';
 import ReactDOM from 'react-dom';
 import GUI, {AppStateHOC} from 'scratch-gui';
@@ -42,7 +43,17 @@ const onStorageInit = storageInstance => {
 const guiProps = {
     onStorageInit,
     isScratchDesktop: true,
-    projectId: defaultProjectId
+    projectId: defaultProjectId,
+    showTelemetryModal: (typeof ipcRenderer.sendSync('getTelemetryDidOptIn')) !== 'boolean',
+    onTelemetryModalOptIn: () => {
+        ipcRenderer.send('setTelemetryDidOptIn', true);
+    },
+    onTelemetryModalOptOut: () => {
+        ipcRenderer.send('setTelemetryDidOptIn', false);
+    },
+    onProjectTelemetryEvent: (event, metadata) => {
+        ipcRenderer.send(event, metadata);
+    }
 };
 const wrappedGui = React.createElement(WrappedGui, guiProps);
 ReactDOM.render(wrappedGui, appTarget);