Subs dashboard perf

Caching older (at least 16 days) Stripe invoices in analytics
collection, which will be updated once a day via the analytics server
cron job.

app/models/AnalyticsStripeInvoice.coffee

@@ -0,0 +1,6 @@
+CocoModel = require './CocoModel'
+
+module.exports = class AnalyticsStripeInvoice extends CocoModel
+  @className: 'AnalyticsStripeInvoice'
+  @schema: require 'schemas/models/analytics_stripe_invoice'
+  urlRoot: '/db/analytics.stripe.invoice'

app/schemas/models/analytics_stripe_invoice.coffee

@@ -0,0 +1,14 @@
+c = require './../schemas'
+
+AnalyticsStripeInvoiceSchema = c.object {
+  title: 'Analytics Stripe Invoice'
+}
+
+_.extend AnalyticsStripeInvoiceSchema.properties,
+  _id: {type: 'string'}
+  date: {type: 'integer'}
+  properties: {type: 'object'}
+
+c.extendBasicProperties AnalyticsStripeInvoiceSchema, 'analytics.stripe.invoice'
+
+module.exports = AnalyticsStripeInvoiceSchema

app/views/admin/AnalyticsSubscriptionsView.coffee

@@ -265,36 +265,58 @@ module.exports = class AnalyticsSubscriptionsView extends RootView
 
   getInvoices: (done) ->
     invoices = {}
-    nextBatch = (starting_after, done) =>
+
+    addInvoice = (invoice) =>
+      return unless invoice.paid
+      return unless invoice.subscription
+      return unless invoice.total > 0
+      return unless invoice.lines?.data?[0]?.plan?.id is 'basic'
+      invoices[invoice.id] =
+        customerID: invoice.customer
+        subscriptionID: invoice.subscription
+        date: new Date(invoice.date * 1000)
+      invoices[invoice.id].userID = invoice.lines.data[0].metadata.id if invoice.lines?.data?[0]?.metadata?.id
+
+    getLiveInvoices = (ending_before, done) =>
+
+      nextBatch = (ending_before, done) =>
+        @updateFetchDataState "Fetching invoices #{Object.keys(invoices).length}..."
+        options =
+          url: '/db/subscription/-/stripe_invoices'
+          method: 'POST'
+          data: {options: {ending_before: ending_before, limit: 100}}
+        options.error = (model, response, options) =>
+          return if @destroyed
+          console.error 'Failed to get live invoices', response
+        options.success = (invoiceData, response, options) =>
+          return if @destroyed
+          addInvoice(invoice) for invoice in invoiceData.data
+          if invoiceData.has_more
+            return nextBatch(invoiceData.data[0].id, done)
+          else
+            invoices = (invoice for invoiceID, invoice of invoices)
+            invoices.sort (a, b) -> if a.date > b.date then -1 else 1
+            return done(invoices)
+        @supermodel.addRequestResource('get_live_invoices', options, 0).load()
+
+      nextBatch ending_before, done
+
+    getAnalyticsInvoices = (done) =>
       @updateFetchDataState "Fetching invoices #{Object.keys(invoices).length}..."
       options =
-        url: '/db/subscription/-/stripe_invoices'
-        method: 'POST'
-        data: {options: {limit: 100}}
-      options.data.options.starting_after = starting_after if starting_after
+        url: '/db/analytics_stripe_invoice/-/all'
+        method: 'GET'
       options.error = (model, response, options) =>
         return if @destroyed
-        console.error 'Failed to get invoices', response
-      options.success = (invoiceData, response, options) =>
+        console.error 'Failed to get analytics stripe invoices', response
+      options.success = (docs, response, options) =>
         return if @destroyed
-        for invoice in invoiceData.data
-          continue unless invoice.paid
-          continue unless invoice.subscription
-          continue unless invoice.total > 0
-          continue unless invoice.lines?.data?[0]?.plan?.id is 'basic'
-          invoices[invoice.id] =
-            customerID: invoice.customer
-            subscriptionID: invoice.subscription
-            date: new Date(invoice.date * 1000)
-          invoices[invoice.id].userID = invoice.lines.data[0].metadata.id if invoice.lines?.data?[0]?.metadata?.id
-        if invoiceData.has_more
-          return nextBatch(invoiceData.data[invoiceData.data.length - 1].id, done)
-        else
-          invoices = (invoice for invoiceID, invoice of invoices)
-          invoices.sort (a, b) -> if a.date > b.date then -1 else 1
-          return done(invoices)
-      @supermodel.addRequestResource('get_invoices', options, 0).load()
-    nextBatch null, done
+        docs.sort (a, b) -> b.date - a.date
+        addInvoice(doc.properties) for doc in docs
+        getLiveInvoices(docs[0]._id, done)
+      @supermodel.addRequestResource('get_analytics_invoices', options, 0).load()
+
+    getAnalyticsInvoices(done)
 
   getRecipientSubscriptions: (sponsors, done) ->
     @updateFetchDataState "Fetching recipient subscriptions..."
@@ -304,6 +326,7 @@ module.exports = class AnalyticsSubscriptionsView extends RootView
         subscriptionsToFetch.push
           customerID: user.stripe.customerID
           subscriptionID: recipient.subscriptionID
+    return done([]) if _.isEmpty subscriptionsToFetch
     options =
       url: '/db/subscription/-/stripe_subscriptions'
       method: 'POST'

scripts/analytics/insertStripeAnalytics.js

@@ -0,0 +1,86 @@
+// Insert older (-16 days) Stripe invoices into coco database analytics collection
+
+if (process.argv.length !== 4) {
+  log("Usage: node <script> <Stripe API key> <mongo connection Url>");
+  process.exit();
+}
+
+var scriptStartTime = new Date();
+var stripeAPIKey = process.argv[2];
+var mongoConnUrl = process.argv[3];
+var stripe = require("stripe")(stripeAPIKey);
+var MongoClient = require('mongodb').MongoClient;
+
+getInvoices(function(invoices) {
+  log("Invoice count: " + invoices.length);
+  insertInvoices(invoices, function() {
+    log("Script runtime: " + (new Date() - scriptStartTime));
+    process.exit(0);
+  });
+});
+
+function log(str) {
+  console.log(new Date().toISOString() + " " + str);
+}
+
+function getInvoices(done) {
+  var sixteenDaysAgo = new Date()
+  sixteenDaysAgo.setUTCDate(sixteenDaysAgo.getUTCDate() - 16);
+  invoiceMaxTimestamp = Math.floor(sixteenDaysAgo.getTime() / 1000);
+  var options = {limit: 100, date: {lt: invoiceMaxTimestamp}};
+  var invoices = [];
+
+  getInvoicesHelper = function(options, done) {
+    // log("getInvoicesHelper " + invoices.length + " " + options.starting_after);
+    stripe.invoices.list(options, function (err, result) {
+      if (err) {
+        console.log(err);
+        return;
+      }
+      invoices = invoices.concat(result.data);
+      if (result.has_more) {
+        options.starting_after = result.data[result.data.length - 1].id
+        getInvoicesHelper(options, done);
+      }
+      else {
+        done(invoices);
+      }
+    });
+  };
+  getInvoicesHelper(options, done);
+}
+
+function insertInvoices(invoices, done) {
+  var docs = [];
+  for (var i = 0; i < invoices.length; i++) {
+    docs.push({
+      _id: invoices[i].id,
+      date: invoices[i].date,
+      properties: invoices[i]
+    });
+  }
+
+  MongoClient.connect(mongoConnUrl, function (err, db) {
+    if (err) {
+      console.log(err);
+      return done();
+    }
+
+    insertInvoicesHelper = function() {
+      var doc = docs.pop();
+      if (!doc) {
+        db.close();
+        return done();
+      }
+      db.collection('analytics.stripe.invoices').save(doc, function(err, result) {
+        if (err) {
+          console.log(err);
+          db.close();
+          return done();
+        }
+        insertInvoicesHelper();
+      });
+    };
+    insertInvoicesHelper();
+  });
+}

server/analytics/AnalyticsStripeInvoice.coffee

@@ -0,0 +1,9 @@
+mongoose = require 'mongoose'
+
+AnalyticsStripeInvoiceSchema = new mongoose.Schema({
+  _id: String
+  date: Number
+  properties: mongoose.Schema.Types.Mixed
+}, {strict: false})
+
+module.exports = AnalyticsStripeInvoice = mongoose.model('analytics.stripe.invoice', AnalyticsStripeInvoiceSchema)

server/analytics/analytics_stripe_invoice_handler.coffee

@@ -0,0 +1,20 @@
+Handler = require '../commons/Handler'
+AnalyticsStripeInvoice = require './AnalyticsStripeInvoice'
+
+class AnalyticsStripeInvoiceHandler extends Handler
+  modelClass: AnalyticsStripeInvoice
+  jsonSchema: require '../../app/schemas/models/analytics_stripe_invoice'
+
+  hasAccess: (req) -> req.user?.isAdmin()
+
+  getByRelationship: (req, res, args...) ->
+    return @sendForbiddenError(res) unless @hasAccess(req)
+    return @getAll(req, res) if args[1] is 'all'
+    super(arguments...)
+
+  getAll: (req, res) ->
+    AnalyticsStripeInvoice.find {}, (err, docs) =>
+      return @sendDatabaseError(res, err) if err
+      @sendSuccess(res, docs)
+
+module.exports = new AnalyticsStripeInvoiceHandler()

server/commons/mapping.coffee

@@ -2,6 +2,7 @@ module.exports.handlers =
   'analytics_log_event': 'analytics/analytics_log_event_handler'
   'analytics_perday': 'analytics/analytics_perday_handler'
   'analytics_string': 'analytics/analytics_string_handler'
+  'analytics_stripe_invoice': 'analytics/analytics_stripe_invoice_handler'
   # TODO: Disabling this until we know why our app servers CPU grows out of control.
   # 'analytics_users_active': 'analytics/analytics_users_active_handler'
   'article': 'articles/article_handler'

server/payments/subscription_handler.coffee

@@ -45,32 +45,8 @@ class SubscriptionHandler extends Handler
     # console.log 'subscription_handler getStripeInvoices'
     return @sendForbiddenError(res) unless req.user?.isAdmin()
 
-    # TODO: this caching mechanism doesn't work in production (multiple calls or app servers?)
-    # TODO: cache older invoices in the analytics database instead
-    # @oldInvoices ?= {}
-    # buildInvoicesFromCache = (newInvoices) =>
-    #   data = (invoice for invoiceID, invoice of @oldInvoices)
-    #   data = data.concat(newInvoices)
-    #   data.sort (a, b) -> if a.date > b.date then -1 else 1
-    #   {has_more: false, data: data}
-    # oldInvoiceCutoffDays = 16 # Dependent on Stripe subscription payment retries
-    # oldInvoiceCutoffDate = new Date()
-    # oldInvoiceCutoffDate.setUTCDate(oldInvoiceCutoffDate.getUTCDate() - oldInvoiceCutoffDays)
     stripe.invoices.list req.body.options, (err, invoices) =>
       return @sendDatabaseError(res, err) if err
-      # newInvoices = []
-      # for invoice, i in invoices.data
-      #   if new Date(invoice.date * 1000) < oldInvoiceCutoffDate
-      #     if invoice.id of @oldInvoices
-      #       # Rest of the invoices should be cached, return from cache
-      #       cachedInvoices = buildInvoicesFromCache(newInvoices)
-      #       return @sendSuccess(res, cachedInvoices)
-      #     else
-      #       # Cache older invoices
-      #       @oldInvoices[invoice.id] = invoice
-      #   else
-      #     # Keep track of new invoices for this page of invoices
-      #     newInvoices.push(invoice)
       @sendSuccess(res, invoices)
 
   getStripeSubscriptions: (req, res) ->