Update revenue analytics with payment.prepaidID

app/templates/admin/analytics.jade

@@ -159,7 +159,7 @@ block content
       tr
         th Day
         for group in revenueGroups
-          th= group.replace('DRR ', '')
+          th= group.replace('DRR ', 'Daily ')
       each entry in revenue
         tr
           td= entry.day

app/views/admin/AnalyticsView.coffee

@@ -88,13 +88,13 @@ module.exports = class AnalyticsView extends RootView
         dayGroupCountMap = {}
         for dailyRevenue in data
           dayGroupCountMap[dailyRevenue.day] ?= {}
-          dayGroupCountMap[dailyRevenue.day]['Daily'] = 0
+          dayGroupCountMap[dailyRevenue.day]['Daily Total'] = 0
           for group, val of dailyRevenue.groups
             groupMap[group] = true
             dayGroupCountMap[dailyRevenue.day][group] = val
-            dayGroupCountMap[dailyRevenue.day]['Daily'] += val
+            dayGroupCountMap[dailyRevenue.day]['Daily Total'] += val
         @revenueGroups = Object.keys(groupMap)
-        @revenueGroups.push 'Daily'
+        @revenueGroups.push 'Daily Total'
         # Build list of recurring revenue entries, where each entry is a day of individual group values
         @revenue = []
         for day of dayGroupCountMap
@@ -604,12 +604,12 @@ module.exports = class AnalyticsView extends RootView
       points = @createLineChartPoints(days, data)
       @revenueChartLines.push
         points: points
-        description: group.replace('DRR ', '')
+        description: group.replace('DRR ', 'Daily ')
         lineColor: @lineColors[colorIndex++ % @lineColors.length]
         strokeWidth: 1
         min: 0
         max: _.max(points, 'y').y
-        showYScale: group in ['Daily', 'Monthly']
+        showYScale: group in ['Daily Total', 'Monthly']
-      dailyMax = _.max(points, 'y').y if group is 'Daily'
+      dailyMax = _.max(points, 'y').y if group is 'Daily Total'
       for line in @revenueChartLines when line.description isnt 'Monthly'
         line.max = dailyMax

scripts/analytics/mongodb/queries/insertAnalyticsRecurringRevenue.js

@@ -1,10 +1,13 @@
+/* global printjson */
+/* global db */
 // Insert per-day recurring revenue counts into analytics.perdays collection
 
 // Usage:
 // mongo <address>:<port>/<database> <script file> -u <username> -p <password>
 
+// TODO: Investigate school sales amount == zero. Manual input error?
+
 try {
-  logDB = new Mongo("localhost").getDB("analytics")
   var scriptStartTime = new Date();
   var analyticsStringCache = {};
 
@@ -26,6 +29,7 @@ try {
   for (var event in recurringRevenueCounts) {
     for (var day in recurringRevenueCounts[event]) {
       if (today === day) continue; // Never save data for today because it's incomplete
+      // print(event, day, recurringRevenueCounts[event][day]);
       insertEventCount(event, day, recurringRevenueCounts[event][day]);
     }
   }
@@ -41,11 +45,13 @@ function getRecurringRevenueCounts(startDay) {
   if (!startDay) return {};
 
   var dailyRevenueCounts = {};
+  var day;
+  var prepaidIDs = [];
+  var prepaidDayAmountMap = {};
   var startObj = objectIdWithTimestamp(ISODate(startDay + "T00:00:00.000Z"));
   var cursor = db.payments.find({_id: {$gte: startObj}});
   while (cursor.hasNext()) {
     var doc = cursor.next();
-    var day;
     if (doc.created) {
       day = doc.created.substring(0, 10);
     }
@@ -55,7 +61,15 @@ function getRecurringRevenueCounts(startDay) {
 
     if (doc.service === 'ios' || doc.service === 'bitcoin') continue;
 
-    if (doc.productID && doc.productID.indexOf('gems_') === 0) {
+    if (doc.prepaidID) {
+      if (prepaidDayAmountMap[doc.prepaidID.valueOf()]) {
+        console.log("ERROR! prepaid", doc.prepaidID.valueOf(), "attached to multiple payments", doc._id.valueOf(), prepaidDayAmountMap[doc.prepaidID.valueOf()]);
+        return {}; 
+      }
+      prepaidDayAmountMap[doc.prepaidID.valueOf()] = {day: day, amount: doc.amount};
+      prepaidIDs.push(doc.prepaidID);
+    }
+    else if (doc.productID && doc.productID.indexOf('gems_') === 0) {
       if (!dailyRevenueCounts['DRR gems']) dailyRevenueCounts['DRR gems'] = {};
       if (!dailyRevenueCounts['DRR gems'][day]) dailyRevenueCounts['DRR gems'][day] = 0;
       dailyRevenueCounts['DRR gems'][day] += doc.amount
@@ -87,6 +101,22 @@ function getRecurringRevenueCounts(startDay) {
     // }
   }
 
+  // Add revenue from prepaids connected to payments
+  cursor = db.prepaids.find({_id: {$in: prepaidIDs}});
+  while (cursor.hasNext()) {
+    doc = cursor.next();
+    if (prepaidDayAmountMap[doc._id.valueOf()]) {
+      day = prepaidDayAmountMap[doc._id.valueOf()].day;
+      var amount = prepaidDayAmountMap[doc._id.valueOf()].amount;
+      if (doc.type === 'course' || doc.type === 'terminal_subscription') {
+        var revenueType = doc.type === 'course' ? 'DRR school sales' : 'DRR monthly subs';
+        if (!dailyRevenueCounts[revenueType]) dailyRevenueCounts[revenueType] = {};
+        if (!dailyRevenueCounts[revenueType][day]) dailyRevenueCounts[revenueType][day] = 0;
+        dailyRevenueCounts[revenueType][day] += amount;
+      }
+    }
+  }
+
   return dailyRevenueCounts;
 }
 
@@ -156,10 +186,10 @@ function insertEventCount(event, day, count) {
   // analytics.perdays schema in server/analytics/AnalyticsPeryDay.coffee
   day = day.replace(/-/g, '');
 
+  var results;
   var eventID = getAnalyticsString(event);
   var filterID = getAnalyticsString('all');
 
-  var startObj = objectIdWithTimestamp(new Date(startDay + "T00:00:00.000Z"));
   var queryParams = {$and: [{d: day}, {e: eventID}, {f: filterID}]};
   var doc = db['analytics.perdays'].findOne(queryParams);
   if (doc && doc.c === count) return;
@@ -167,7 +197,7 @@ function insertEventCount(event, day, count) {
   if (doc && doc.c !== count) {
     // Update existing count, assume new one is more accurate
     // log("Updating count in db for " + day + " " + event + " " + doc.c + " => " + count);
-    var results = db['analytics.perdays'].update(queryParams, {$set: {c: count}});
+    results = db['analytics.perdays'].update(queryParams, {$set: {c: count}});
     if (results.nMatched !== 1 && results.nModified !== 1) {
       log("ERROR: update event count failed");
       printjson(results);
@@ -175,7 +205,7 @@ function insertEventCount(event, day, count) {
   }
   else {
     var insertDoc = {d: day, e: eventID, f: filterID, c: count};
-    var results = db['analytics.perdays'].insert(insertDoc);
+    results = db['analytics.perdays'].insert(insertDoc);
     if (results.nInserted !== 1) {
       log("ERROR: insert event failed");
       printjson(results);

scripts/attachPrepaidsToPayments.js

@@ -0,0 +1,185 @@
+/* global process */
+// Attach missing prepaidID properties to Payments
+
+// TODO: investigate payments that match multiple prepaids
+// TODO: investigate payments that don't match any prepaids
+
+// Payments and Stripe charges are tightly bound via payment.stripe.chargeID
+// Stripe charges and prepaids are loosely bound via maxRedeemers, type, and user
+
+// Steps:
+// 1. Find paid prepaids for courses and subscriptions
+// 2. Find paid prepaids disconnected from payments
+// 3. Find Stripe charge payments for disconnected prepaid users
+// 4. Set payment.prepaidID for charges matching prepaids
+
+if (process.argv.length !== 4) {
+  log("Usage: node <script> <Stripe API key> <mongo connection Url>");
+  process.exit();
+}
+
+var scriptStartTime = new Date();
+var stripeAPIKey = process.argv[2];
+var mongoConnUrl = process.argv[3];
+var stripe = require("stripe")(stripeAPIKey);
+var MongoClient = require('mongodb').MongoClient;
+var ObjectId = require('mongodb').ObjectId;
+
+MongoClient.connect(mongoConnUrl, function (err, db) {
+  if (err) {
+    console.log(err);
+    return;
+  }
+  // Find paid prepaids for courses and subscriptions
+  var prepaidTypes = ['course', 'terminal_subscription'];
+  getPaidPrepaids(db, prepaidTypes, function(prepaidIDs, prepaidPaymentMap, userPrepaidsMap) {
+    log("Paid prepaids: " + prepaidIDs.length);
+
+    // Find paid prepaids disconnected from payments
+    getDisconnectedPrepaidUsers(db, prepaidIDs, prepaidPaymentMap, function(missingUserIDs) {
+      log("Prepaids with no payment: " + missingUserIDs.length);
+
+      // Find Stripe charge payments for disconnected prepaid users
+      getDisconnectedCharges(db, missingUserIDs, function(chargePaymentMap, disconnectedCharges) {
+        log("Disconnected charges: " + disconnectedCharges.length);
+
+        // Set payment.prepaidID for charges matching prepaids 
+        attachPrepaidsToPayments(db, chargePaymentMap, disconnectedCharges, userPrepaidsMap, function() {
+          db.close();
+          log("Script runtime: " + (new Date() - scriptStartTime));
+        });
+      });
+    });
+  });
+});
+
+function getPaidPrepaids(db, prepaidTypes, done) {
+  db.collection('prepaids').find({type: {$in: prepaidTypes}}, {creator: 1, maxRedeemers: 1, properties: 1, type: 1}).toArray(function(err, prepaids) {
+    if (err) {
+      console.log(err);
+      return done([], {}, {});
+    }
+    var prepaidIDs = [];
+    var prepaidPaymentMap = {};
+    var userPrepaidsMap = {};
+    for (var i = 0; i < prepaids.length; i++) {
+      if (!(prepaids[i].properties && prepaids[i].properties.trialRequestID)) {
+        var userID = prepaids[i].creator.valueOf();
+        prepaidIDs.push(new ObjectId(prepaids[i]._id));
+        prepaidPaymentMap[prepaids[i]._id] = {prepaid: prepaids[i]}
+        if (!userPrepaidsMap[userID]) userPrepaidsMap[userID] = [];
+        userPrepaidsMap[userID].push(prepaids[i]);
+      }
+    }
+    return done(prepaidIDs, prepaidPaymentMap, userPrepaidsMap);
+  });
+}
+
+function getDisconnectedPrepaidUsers(db, prepaidIDs, prepaidPaymentMap, done) {
+  db.collection('payments').find({prepaidID: {$in: prepaidIDs}}).toArray(function (err, payments) {
+    if (err) {
+      console.log(err);
+      return done([]);
+    }
+    for (var i = 0; i < payments.length; i++) {
+      prepaidPaymentMap[payments[i].prepaidID].payment = payments[i];
+    }
+    var missingUserIDs = [];
+    for (var prepaidID in prepaidPaymentMap) {
+      if (!prepaidPaymentMap[prepaidID].payment) {
+        missingUserIDs.push(prepaidPaymentMap[prepaidID].prepaid.creator);
+      }
+    }
+    return done(missingUserIDs);
+  });
+}
+
+function getDisconnectedCharges(db, missingUserIDs, done) {
+  db.collection('payments').find({$and: [{purchaser: {$in: missingUserIDs}}, {service: 'stripe'}]}, {amount: 1, prepaidID: 1, stripe: 1}).toArray(function (err, payments) {
+    if (err) {
+      console.log(err);
+      return done({}, []);
+    }
+    var chargePaymentMap = {};
+    var disconnectedCharges = [];
+    for (var i = 0; i < payments.length; i++) {
+      if (!payments[i].prepaidID && payments[i].stripe && payments[i].stripe.chargeID) {
+        disconnectedCharges.push(payments[i].stripe.chargeID);
+        chargePaymentMap[payments[i].stripe.chargeID] = payments[i];
+      }
+    }
+    return done(chargePaymentMap, disconnectedCharges);
+  });
+}
+
+function attachPrepaidsToPayments(db, chargePaymentMap, disconnectedCharges, userPrepaidsMap, done) {
+  var processCharge = function(disconnectedCharge, done) {
+    stripe.charges.retrieve(disconnectedCharge, function(err, charge) {
+      var prepaid, payment;
+      if (err) {
+        console.log("Skipping error", disconnectedCharge);
+      }
+      else if (!charge) {
+        console.log("Skipping not found", disconnectedCharge);
+      }
+      else if (!charge.metadata) {
+        console.log('Skipping no metadata', disconnectedCharge);
+      }
+      else {
+        var chargeUserID = charge.metadata.userID;
+        var matches = [];
+        for (var i = 0; i < userPrepaidsMap[chargeUserID].length; i++) {
+          var currPrepaid = userPrepaidsMap[chargeUserID][i];
+          if (charge.metadata.type === currPrepaid.type && parseInt(charge.metadata.maxRedeemers) === parseInt(currPrepaid.maxRedeemers)) {
+            matches.push(currPrepaid);
+          }
+        }
+        if (matches.length === 1) {
+          payment = chargePaymentMap[charge.id];
+          prepaid = matches[0];
+          console.log("Saving prepaid", prepaid._id.valueOf(), 'to payment', payment._id.valueOf(), 'for user', chargeUserID, 'amount', chargePaymentMap[charge.id].amount);
+        }
+        else {
+          console.log("No match", matches.length, 'user', chargeUserID, 'payment', chargePaymentMap[charge.id]._id.valueOf());
+          if (matches.length > 1) console.log(matches);
+        }
+      }
+      
+      var next = function() {
+        disconnectedCharges.shift();
+        if (disconnectedCharges.length > 0) {
+          return processCharge(disconnectedCharges[0], done);
+        }
+        return done();
+      }
+
+      if (payment && prepaid) {
+        db.collection('payments').update({_id: new ObjectId(payment._id)}, {$set: {prepaidID: new ObjectId(prepaid._id)}}, function (err, response) {
+          if (err) {
+            console.log(err);
+            return done();
+          }
+          if (!response || !response.result || response.result.nModified < 1) {
+            console.log("Payment", payment._id, "not modified with prepaid", prepaid._id);
+            console.log(response.result || response);
+            return done();
+          }
+          return next();
+        });
+      }
+      else {
+        return next();
+      } 
+    });
+  }
+  processCharge(disconnectedCharges[0], function() {
+    return done();
+  });
+}
+
+// *** Helper functions ***
+
+function log(str) {
+  console.log(new Date().toISOString() + " " + str);
+}
+