Fix analytics MAUs and campaign paid users

app/templates/admin/analytics.jade

@@ -67,7 +67,7 @@ block content
         h3 Active Classes 365 days
         .active-classes-chart-365.line-chart-container
 
-        h1#active-classes-table Active Classes
+        h1 Active Classes
         table.table.table-striped.table-condensed
           tr
             th Day
@@ -93,7 +93,35 @@ block content
         h3 Monthly Recurring Revenue 365 days
         .recurring-monthly-revenue-chart-365.line-chart-container
 
-        h1#recurring-revenue-table Recurring Revenue
+        .school-sales
+          h3 School Sales
+          if view.schoolSales
+            table.table.table-striped.table-condensed
+              tr
+                th Amount
+                th(style='min-width:85px;') Created
+                th PaymentID
+                th PrepaidID
+                th Description
+                th Email
+                th School
+              each val, i in view.schoolSales
+                tr
+                  td $#{val.amount / 100}
+                  td= new Date(val.created).toISOString().substring(0, 10)
+                  td= val._id
+                  td= val.prepaidID
+                  td= val.description
+                  if val.user
+                    td= val.user.emailLower
+                    td= val.user.schoolName
+                  else
+                    td
+                    td
+          else
+            div Loading ...
+
+        h1 Recurring Revenue
         table.table.table-striped.table-condensed
           tr
             th(style='min-width:85px;') Day
@@ -106,22 +134,22 @@ block content
                 td $#{(val / 100).toFixed(2)}
 
       .tab-pane#tab_classroom
-        h3#classroom-daus-graph Classroom Daily Active Users 90 days
+        h3 Classroom Daily Active Users 90 days
         .small Paid student: user.coursePrepaidID set and prepaid.properties.trialRequestID NOT set
         .small Trial student: user.coursePrepaidID set and prepaid.properties.trialRequestID set
         .small Free student: not paid, not trial
         .classroom-daily-active-users-chart-90.line-chart-container
 
-        h3#classroom-maus-graph Classroom Monthly Active Users 90 days
+        h3 Classroom Monthly Active Users 90 days
         .classroom-monthly-active-users-chart-90.line-chart-container
 
-        h3#classroom-daus-graph Classroom Daily Active Users 365 days
+        h3 Classroom Daily Active Users 365 days
         .classroom-daily-active-users-chart-365.line-chart-container
 
-        h3#classroom-maus-graph Classroom Monthly Active Users 365 days
+        h3 Classroom Monthly Active Users 365 days
         .classroom-monthly-active-users-chart-365.line-chart-container
 
-        h3#enrollments-graph Enrollments Issued and Redeemed 90 days
+        h3 Enrollments Issued and Redeemed 90 days
         .paid-courses-chart.line-chart-container
 
         #furthest-course
@@ -191,7 +219,7 @@ block content
           else
             div Loading ...
 
-        #school-sales
+        .school-sales
           h3 School Sales
           if view.schoolSales
             table.table.table-striped.table-condensed
@@ -205,7 +233,7 @@ block content
                 th School
               each val, i in view.schoolSales
                 tr
-                  td $#{Math.round(val.amount / 100, 2)}
+                  td $#{val.amount / 100}
                   td= new Date(val.created).toISOString().substring(0, 10)
                   td= val._id
                   td= val.prepaidID
@@ -236,7 +264,7 @@ block content
           else
             div Loading ...
 
-        h1#active-users-table Active Users
+        h1 Active Users
         if activeUsers.length > 0
           - var eventNames = [];
             each count, event in activeUsers[0].events
@@ -296,7 +324,7 @@ block content
         h3 Campaign Monthly Active Users 365 days
         .campaign-monthly-active-users-chart-365.line-chart-container
 
-        h1#active-users-table Active Users
+        h1 Active Users
         if activeUsers.length > 0
           - var eventNames = [];
             each count, event in activeUsers[0].events
@@ -321,13 +349,13 @@ block content
 
 
       .tab-pane#tab_campaign_vs_classroom
-        h3#campaign-vs-classroom-paid-maus-recent-graph Campaign vs Classroom Paid Monthly Active Users 90 days
+        h3 Campaign vs Classroom Paid Monthly Active Users 90 days
         .campaign-vs-classroom-monthly-active-users-recent-chart.line-chart-container
 
-        h3#campaign-vs-classroom-paid-maus-graph Campaign vs Classroom Paid Monthly Active Users 365 days
+        h3 Campaign vs Classroom Paid Monthly Active Users 365 days
         .campaign-vs-classroom-monthly-active-users-chart.line-chart-container
 
-        h1#active-users-table Active Users
+        h1 Active Users
         if activeUsers.length > 0
           - var eventNames = [];
             each count, event in activeUsers[0].events

app/views/admin/AnalyticsView.coffee

@@ -158,7 +158,7 @@ module.exports = class AnalyticsView extends RootView
           return -1 if a.created > b.created
           return 0 if a.created is b.created
           1
-        @renderSelectors?('#school-sales')
+        @renderSelectors?('.school-sales')
     }, 0).load()
 
     @supermodel.addRequestResource({

scripts/analytics/mongodb/queries/insertAnalyticsActiveUsers.js

@@ -23,8 +23,8 @@ try {
   var endDay = new Date();
   endDay = endDay.toISOString().substr(0, 10);
 
-  // startDay = '2015-03-01';
+  // startDay = '2015-06-01';
-  // endDay = '2015-06-01';
+  // endDay = '2015-08-01';
 
   var activeUserEvents = ['Finished Signup', 'Started Level'];
 
@@ -59,36 +59,57 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
   // Counts active users per day
   if (!startDay) return {};
   
+  // Faster to request analytics db data in batches of days
+  var dayIncrement = 3;
+  var startDate = new Date(startDay + "T00:00:00.000Z");
+  var interimEndDate  = new Date(startDay + "T00:00:00.000Z");
+  interimEndDate.setUTCDate(interimEndDate.getUTCDate() + dayIncrement);
+  var interimEndDay = interimEndDate.toISOString().substr(0, 10);
+
   var cursor, doc;
 
   log("Finding active user log events..");
-  var startObj = objectIdWithTimestamp(ISODate(startDay + "T00:00:00.000Z"));
-  var endObj = objectIdWithTimestamp(ISODate(endDay + "T00:00:00.000Z"));
-  var queryParams = {$and: [
-    {_id: {$gte: startObj}},
-    {_id: {$lt: endObj}},
-    {'event': {$in: activeUserEvents}}
-  ]};
-  cursor = logDB['log'].find(queryParams);
-
   var campaignUserMap = {};
-  var dayUserMap = {};
+  var days = {};
+  var dayUserActiveMap = {};
   var userIDs = [];
-  while (cursor.hasNext()) {
+  while (startDay < endDay) {
-    doc = cursor.next();
+    var startObj = objectIdWithTimestamp(ISODate(startDay + "T00:00:00.000Z"));
-    var created = doc._id.getTimestamp().toISOString();
+    var endObj = objectIdWithTimestamp(ISODate(interimEndDay + "T00:00:00.000Z"));
-    var day = created.substring(0, 10);
+    var queryParams = {$and: [
-    var user = doc.user.valueOf();
+      {_id: {$gte: startObj}},
-    campaignUserMap[user] = true;
+      {_id: {$lt: endObj}},
-    if (!dayUserMap[day]) dayUserMap[day] = {};
+      {'event': {$in: activeUserEvents}}
-    dayUserMap[day][user] = true;
+    ]};
-    userIDs.push(ObjectId(user));
+    cursor = logDB['log'].find(queryParams);
-    // if (userIDs.length % 100000 === 0) {
+
-    //   log('Users so far: ' + userIDs.length);
+    while (cursor.hasNext()) {
-    // }
+      doc = cursor.next();
+      var created = doc._id.getTimestamp().toISOString();
+      var day = created.substring(0, 10);
+      var user = doc.user.valueOf();
+      days[day] = true;
+      campaignUserMap[user] = true;
+      if (!dayUserActiveMap[day]) dayUserActiveMap[day] = {};
+      dayUserActiveMap[day][user] = true;
+      userIDs.push(ObjectId(user));
+      // if (userIDs.length % 100000 === 0) {
+      //   log('Users so far: ' + userIDs.length);
+      // }
+    }
+    startDate.setUTCDate(startDate.getUTCDate() + dayIncrement);
+    startDay = startDate.toISOString().substr(0, 10);
+    interimEndDate.setUTCDate(interimEndDate.getUTCDate() + dayIncrement);
+    interimEndDay = interimEndDate.toISOString().substr(0, 10);
+    if (interimEndDay.localeCompare(endDay) > 0) {
+      interimEndDay = endDay;
+    }
   }
   log('User count: ' + userIDs.length);
 
+  days = Object.keys(days);
+  days.sort(function (a, b) {return a.localeCompare(b);});
+
   log("Finding classroom members..");
   var classroomUserObjectIds = [];
   var batchSize = 100000;
@@ -112,20 +133,20 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
   // Trial user: prepaid.properties.trialRequestID means access was via trial
   // Free: not paid, not trial
   log("Finding classroom users free/trial/paid status..");
-  var userEventMap = {};
+  var classroomUserEventMap = {};
   var prepaidUsersMap = {};
   var prepaidIDs = [];
   cursor = db.users.find({_id: {$in: classroomUserObjectIds}}, {coursePrepaidID: 1});
   while (cursor.hasNext()) {
     doc = cursor.next();
     if (doc.coursePrepaidID) {
-      userEventMap[doc._id.valueOf()] = 'DAU classroom paid';
+      classroomUserEventMap[doc._id.valueOf()] = 'DAU classroom paid';
       if (!prepaidUsersMap[doc.coursePrepaidID.valueOf()]) prepaidUsersMap[doc.coursePrepaidID.valueOf()] = [];
       prepaidUsersMap[doc.coursePrepaidID.valueOf()].push(doc._id.valueOf()); 
       prepaidIDs.push(doc.coursePrepaidID);
     }
     else {
-      userEventMap[doc._id.valueOf()] = 'DAU classroom free';
+      classroomUserEventMap[doc._id.valueOf()] = 'DAU classroom free';
     }
   }
   cursor = db.prepaids.find({_id: {$in: prepaidIDs}}, {properties: 1});
@@ -133,7 +154,7 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
     doc = cursor.next();
     if (doc.properties && doc.properties.trialRequestID) {
       for (var i = 0; i < prepaidUsersMap[doc._id.valueOf()].length; i++) {
-        userEventMap[prepaidUsersMap[doc._id.valueOf()][i]] = 'DAU classroom trial';
+        classroomUserEventMap[prepaidUsersMap[doc._id.valueOf()][i]] = 'DAU classroom trial';
       }
     }
   }
@@ -141,19 +162,20 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
   // Campaign free/paid
   // Monthly sub: recipient for payment.stripe.subscriptionID == 'basic'
   // Yearly sub: recipient for paymen.stripe.gems == 42000
+  // NOTE: payment.stripe.subscriptionID === basic from 2014-12-03 to 2015-03-13
   // TODO: missing a number of corner cases here (e.g. cancelled sub, purchased via admin)
   var campaignUserIDs = [];
   for (var userID in campaignUserMap) {
     if (campaignUserMap[userID]) campaignUserIDs.push(ObjectId(userID));
   }
   log("Finding campaign paid users..");
-  var dayUserPaidMap = {};
+  var dayCampaignUserPaidMap = {};
   batchSize = 100000;
   for (var j = 0; j < campaignUserIDs.length / batchSize + 1; j++) {
     cursor = db.payments.find({$and: [
       {recipient: {$in: campaignUserIDs.slice(j * batchSize, j * batchSize + batchSize)}}, 
       {$or: [
-        {'stripe.subscriptionID': 'basic'},
+        {$and: [{amount: {$gt: 0}}, {gems: 3500}, {'stripe.subscriptionID': {$exists: true}}]},
         {gems: 42000}
       ]}
     ]}, {created: 1, gems: 1, recipient: 1});
@@ -164,8 +186,8 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
       var numDays = doc.gems === 42000 ? 365 : 30;
       for (var i = 0; i < numDays; i++) {
         day = currentDate.toISOString().substring(0, 10);
-        if (!dayUserPaidMap[day]) dayUserPaidMap[day] = {};
+        if (!dayCampaignUserPaidMap[day]) dayCampaignUserPaidMap[day] = {};
-        dayUserPaidMap[day][userID] = true;
+        dayCampaignUserPaidMap[day][userID] = true;
         currentDate.setUTCDate(currentDate.getUTCDate() + 1);
       }
     }
@@ -174,13 +196,16 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
   log("Calculating DAUs..");
   var activeUsersCounts = {};
   var dailyEventNames = {};
-  for (day in dayUserMap) {
+  var userDayEventMap = {}
-    for (var user in dayUserMap[day]) {
+  for (day in dayUserActiveMap) {
-      var event = userEventMap[user] || (dayUserPaidMap[day] && dayUserPaidMap[day][user] ? 'DAU campaign paid' : 'DAU campaign free');
+    for (var user in dayUserActiveMap[day]) {
+      var event = classroomUserEventMap[user] || (dayCampaignUserPaidMap[day] && dayCampaignUserPaidMap[day][user] ? 'DAU campaign paid' : 'DAU campaign free');
       dailyEventNames[event] = true;
       if (!activeUsersCounts[day]) activeUsersCounts[day] = {};
       if (!activeUsersCounts[day][event]) activeUsersCounts[day][event] = 0;
       activeUsersCounts[day][event]++;
+      if (!userDayEventMap[user]) userDayEventMap[user] = {};
+      userDayEventMap[user][day] = event;
     }
   }
   // printjson(dailyEventNames)
@@ -209,30 +234,27 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
     }
   }
 
-  // Calculate monthly actives for each day, starting when we have enough data
   log("Calculating MAUs..");
-  var days = [];
+  // Calculate monthly actives for each day, starting when we have enough data
-  for (var day in activeUsersCounts) {
+  // TODO: missing log data correction for MAUs
-    days.push(day);
+  for (var user in campaignUserMap) {
-  }
+    // For each day, starting when we have daysInMonth days of prior data
-  days.sort(function (a, b) {return a.localeCompare(b);});
+    for (var i = daysInMonth - 1; i < days.length; i++) {
-  // print('Num days', days.length);
+      var targetMonthlyDay = days[i];
-
+      var eventActiveMap = {}
-  // For each day, starting when we have daysInMonth days of prior data
+      // Find active events for the last daysInMonth days up to the current day
-  for (var i = daysInMonth - 1; i < days.length; i++) {
+      for (var j = i - daysInMonth + 1; j <= i; j++) {
-    // For the last daysInMonth days up to the current day
+        var targetDailyDay = days[j];
-    var targetMonthlyDay = days[i];
+        if (dayUserActiveMap[targetDailyDay][user]) {
-    for (var j = i - daysInMonth + 1; j <= i; j++) {
+          event = userDayEventMap[user][targetDailyDay];
-      var targetDailyDay = days[j];
+          eventActiveMap[event] = true;
-      // For each daily event
-      for (var event in dailyEventNames) {
-        // print(day, event, activeUsersCounts[day][event]);
-        var mauEvent = event.replace('DAU', 'MAU');
-        if (!activeUsersCounts[targetMonthlyDay][mauEvent]) activeUsersCounts[targetMonthlyDay][mauEvent] = 0
-        if (activeUsersCounts[targetDailyDay][event]) {
-          activeUsersCounts[targetMonthlyDay][mauEvent] += activeUsersCounts[targetDailyDay][event];
         }
       }
+      for (var event in eventActiveMap) {
+        var mauEvent = event.replace('DAU', 'MAU');
+        if (!activeUsersCounts[targetMonthlyDay][mauEvent]) activeUsersCounts[targetMonthlyDay][mauEvent] = 0;
+        activeUsersCounts[targetMonthlyDay][mauEvent]++;
+      }
     }
   }