🐛Fix analytics dashboard coursePrepaid use

app/templates/admin/analytics.jade

@@ -267,9 +267,9 @@ block content
         h1 Active Users
         if view.activeUsers.length > 0
           - var eventNames = [];
-            each count, event in view.activeUsers[0].events
+            each event in view.activeUserEventNames
               if event.indexOf('classroom') >= 0
-                - eventNames.push(event)
+                - eventNames.push(event);
           - eventNames.sort(function (a, b) {return a.localeCompare(b);});
           table.table.table-striped.table-condensed
             tr
@@ -322,9 +322,9 @@ block content
         h1 Active Users
         if view.activeUsers.length > 0
           - var eventNames = [];
-            each count, event in view.activeUsers[0].events
+            each event in view.activeUserEventNames
               if event.indexOf('campaign') >= 0
-                - eventNames.push(event)
+                - eventNames.push(event);
           - eventNames.sort(function (a, b) {return a.localeCompare(b);});
           table.table.table-striped.table-condensed
             tr
@@ -346,27 +346,13 @@ block content
 
         h1 Active Users
         if view.activeUsers.length > 0
-          - var eventNames = [];
-            each count, event in view.activeUsers[0].events
-              - eventNames.push(event)
-          - eventNames.sort(function (a, b) {
-            - if (a.indexOf('campaign') == b.indexOf('campaign') || a.indexOf('classroom') == b.indexOf('classroom')) {
-              - return a.localeCompare(b);
-            - }
-            - else if (a.indexOf('campaign') > b.indexOf('campaign')) {
-              - return 1;
-            - }
-            - else {
-              - return -1;
-            - }
-          - });
           table.table.table-striped.table-condensed
             tr
               th(style='min-width:85px;') Day
-              each eventName in eventNames
+              each eventName in view.activeUserEventNames
                 th= eventName
             each activeUser in view.activeUsers
               tr
                 td= activeUser.day
-                each eventName in eventNames
+                each eventName in view.activeUserEventNames
                   td= activeUser.events[eventName] || 0

app/views/admin/AnalyticsView.coffee

@@ -73,6 +73,7 @@ module.exports = class AnalyticsView extends RootView
         # Add campaign/classroom DAU 30-day averages and daily totals
         campaignDauTotals = []
         classroomDauTotals = []
+        eventMap = {}
         for entry in @activeUsers
           day = entry.day
           campaignDauTotal = 0
@@ -82,18 +83,31 @@ module.exports = class AnalyticsView extends RootView
               campaignDauTotal += count
             else if event.indexOf('DAU classroom') >= 0
               classroomDauTotal += count
+            eventMap[event] = true;
           entry.events['DAU campaign total'] = campaignDauTotal
+          eventMap['DAU campaign total'] = true;
           campaignDauTotals.unshift(campaignDauTotal)
           campaignDauTotals.pop() while campaignDauTotals.length > 30
           if campaignDauTotals.length is 30
             entry.events['DAU campaign 30-day average'] = Math.round(_.reduce(campaignDauTotals, (a, b) -> a + b) / 30)
+            eventMap['DAU campaign 30-day average'] = true;
           entry.events['DAU classroom total'] = classroomDauTotal
+          eventMap['DAU classroom total'] = true;
           classroomDauTotals.unshift(classroomDauTotal)
           classroomDauTotals.pop() while classroomDauTotals.length > 30
           if classroomDauTotals.length is 30
             entry.events['DAU classroom 30-day average'] = Math.round(_.reduce(classroomDauTotals, (a, b) -> a + b) / 30)
+            eventMap['DAU classroom 30-day average'] = true;
 
         @activeUsers.sort (a, b) -> b.day.localeCompare(a.day)
+        @activeUserEventNames = Object.keys(eventMap)
+        @activeUserEventNames.sort (a, b) ->
+          if a.indexOf('campaign') is b.indexOf('campaign') or a.indexOf('classroom') is b.indexOf('classroom')
+            a.localeCompare(b)
+          else if a.indexOf('campaign') > b.indexOf('campaign')
+            1
+          else
+            -1
 
         @updateAllKPIChartData()
         @updateActiveUsersChartData()
@@ -183,7 +197,7 @@ module.exports = class AnalyticsView extends RootView
     @supermodel.addRequestResource({
       url: '/db/prepaid/-/courses'
       method: 'POST'
-      data: {project: {maxRedeemers: 1, properties: 1, redeemers: 1}}
+      data: {project: {endDate: 1, maxRedeemers: 1, properties: 1, redeemers: 1}}
       success: (prepaids) =>
         paidDayMaxMap = {}
         paidDayRedeemedMap = {}
@@ -201,14 +215,13 @@ module.exports = class AnalyticsView extends RootView
               redeemDay = redeemer.date.substring(0, 10)
               trialDayRedeemedMap[redeemDay] ?= 0
               trialDayRedeemedMap[redeemDay]++
-          else
+          else if not prepaid.endDate? or new Date(prepaid.endDate) > new Date()
             paidDayMaxMap[day] ?= 0
             paidDayMaxMap[day] += prepaid.maxRedeemers
             for redeemer in prepaid.redeemers
               redeemDay = redeemer.date.substring(0, 10)
               paidDayRedeemedMap[redeemDay] ?= 0
               paidDayRedeemedMap[redeemDay]++
-              
         @dayEnrollmentsMap = {}
         @paidCourseTotalEnrollments = []
         for day, count of paidDayMaxMap
@@ -368,14 +381,17 @@ module.exports = class AnalyticsView extends RootView
 
     # Trim points preceding days
     if points.length and days.length and points[0].day.localeCompare(days[0]) < 0
-      for point, i in points
+      if points[points.length - 1].day.localeCompare(days[0]) < 0
-        if point.day.localeCompare(days[0]) >= 0
+        points = []
-          points.splice(0, i)
+      else
-          break
+        for point, i in points
+          if point.day.localeCompare(days[0]) >= 0
+            points.splice(0, i)
+            break
 
     # Ensure points for each day
     for day, i in days
-      if points.length <= i or points[i].day isnt day
+      if points.length <= i or points[i]?.day isnt day
         prevY = if i > 0 then points[i - 1].y else 0.0
         points.splice i, 0,
           day: day
@@ -648,16 +664,14 @@ module.exports = class AnalyticsView extends RootView
     dailyMax = 0
 
     data = []
-    total = 0
     for entry in @paidCourseTotalEnrollments
-      total += entry.count
       data.push
         day: entry.day
-        value: total
+        value: entry.count
     points = @createLineChartPoints(days, data)
     @enrollmentsChartLines.push
       points: points
-      description: 'Total paid enrollments issued'
+      description: 'Paid enrollments issued'
       lineColor: @lineColors[colorIndex++ % @lineColors.length]
       strokeWidth: 1
       min: 0
@@ -666,16 +680,14 @@ module.exports = class AnalyticsView extends RootView
     dailyMax = _.max([dailyMax, _.max(points, 'y').y])
 
     data = []
-    total = 0
     for entry in @paidCourseRedeemedEnrollments
-      total += entry.count
       data.push
         day: entry.day
-        value: total
+        value: entry.count
     points = @createLineChartPoints(days, data)
     @enrollmentsChartLines.push
       points: points
-      description: 'Total paid enrollments redeemed'
+      description: 'Paid enrollments redeemed'
       lineColor: @lineColors[colorIndex++ % @lineColors.length]
       strokeWidth: 1
       min: 0
@@ -684,16 +696,14 @@ module.exports = class AnalyticsView extends RootView
     dailyMax = _.max([dailyMax, _.max(points, 'y').y])
 
     data = []
-    total = 0
     for entry in @trialCourseTotalEnrollments
-      total += entry.count
       data.push
         day: entry.day
-        value: total
+        value: entry.count
-    points = @createLineChartPoints(days, data)
+    points = @createLineChartPoints(days, data, true)
     @enrollmentsChartLines.push
       points: points
-      description: 'Total trial enrollments issued'
+      description: 'Trial enrollments issued'
       lineColor: @lineColors[colorIndex++ % @lineColors.length]
       strokeWidth: 1
       min: 0
@@ -702,16 +712,14 @@ module.exports = class AnalyticsView extends RootView
     dailyMax = _.max([dailyMax, _.max(points, 'y').y])
 
     data = []
-    total = 0
     for entry in @trialCourseRedeemedEnrollments
-      total += entry.count
       data.push
         day: entry.day
-        value: total
+        value: entry.count
     points = @createLineChartPoints(days, data)
     @enrollmentsChartLines.push
       points: points
-      description: 'Total trial enrollments redeemed'
+      description: 'Trial enrollments redeemed'
       lineColor: @lineColors[colorIndex++ % @lineColors.length]
       strokeWidth: 1
       min: 0

scripts/analytics/mongodb/insertAnalyticsActiveClasses.js

@@ -6,51 +6,49 @@
 // Usage:
 // mongo <address>:<port>/<database> <script file> -u <username> -p <password>
 
-try {
+// TODO: Does not handle course prepaid updates on a user
-  var logDB = new Mongo("localhost").getDB("analytics")
+// TODO: Does not handle class membership changes
-  var scriptStartTime = new Date();
-  var analyticsStringCache = {};
 
-  var minClassSize = 12;
+// TODO: Investigate abrupt trial drop off at 4/1/16.  Showed up when fixing coursePrepaid.
-  var minActiveCount = 6;
 
-  var eventNamePaid = 'Active classes paid';
+var analyticsDB = new Mongo("localhost").getDB("analytics")
-  var eventNameTrial = 'Active classes trial';
+var scriptStartTime = new Date();
-  var eventNameFree = 'Active classes free';
+var analyticsStringCache = {};
 
-  var numDays = 40;
+var minClassSize = 12;
-  var daysInMonth = 30;
+var minActiveCount = 6;
 
-  var startDay = new Date();
+var eventNamePaid = 'Active classes paid';
-  var today = startDay.toISOString().substr(0, 10);
+var eventNameTrial = 'Active classes trial';
-  startDay.setUTCDate(startDay.getUTCDate() - numDays);
+var eventNameFree = 'Active classes free';
-  startDay = startDay.toISOString().substr(0, 10);
 
-  log("Today is " + today);
+var numDays = 40;
-  log("Start day is " + startDay);
+var daysInMonth = 30;
 
-  log("Getting active class counts..");
+var startDay = new Date();
-  var activeClassCounts = getActiveClassCounts(startDay);
+var today = startDay.toISOString().substr(0, 10);
-  // printjson(activeClassCounts);
+startDay.setUTCDate(startDay.getUTCDate() - numDays);
-  log("Inserting active class counts..");
+startDay = startDay.toISOString().substr(0, 10);
-  for (var event in activeClassCounts) {
+
-    for (var day in activeClassCounts[event]) {
+log("Today is " + today);
-      if (today === day) continue; // Never save data for today because it's incomplete
+log("Start day is " + startDay);
-      // print(event, day, activeClassCounts[event][day]);
+
-      insertEventCount(event, day, activeClassCounts[event][day]);
+log("Getting active class counts..");
-    }
+var activeClassCounts = getActiveClassCounts(startDay);
+// printjson(activeClassCounts);
+// log("Inserting active class counts..");
+for (var event in activeClassCounts) {
+  for (var day in activeClassCounts[event]) {
+    if (today === day) continue; // Never save data for today because it's incomplete
+    // print(event, day, activeClassCounts[event][day]);
+    insertEventCount(event, day, activeClassCounts[event][day]);
   }
+}
 
-  log("Script runtime: " + (new Date() - scriptStartTime));
+log("Script runtime: " + (new Date() - scriptStartTime));
-}
-catch(err) {
-  log("ERROR: " + err);
-  printjson(err);
-}
 
 function getActiveClassCounts(startDay) {
   // Tally active classes per day, for paid, trial, and free
-  // TODO: does not handle class membership changes
 
   if (!startDay) return {};
 
@@ -60,7 +58,7 @@ function getActiveClassCounts(startDay) {
   // paid: at least one paid member
   // trial: not paid, at least one trial member
   // free: not paid, not free trial
-  // user.coursePrepaidID set means access to paid courses
+  // user.coursePrepaidID or user.coursePrepaid set means access to paid courses
   // prepaid.properties.trialRequestID means access was via trial
 
   // Find classroom users
@@ -86,30 +84,46 @@ function getActiveClassCounts(startDay) {
       }
     }
   }
+  // log("DEBUG: Classroom users: " + classroomUserIDs.length);
 
   log("Find user types..");
-  var userEventMap = {};
+  var userEventEndDateMap = {};
   var prepaidUsersMap = {};
   var prepaidIDs = [];
-  cursor = db.users.find({_id: {$in: classroomUserObjectIds}}, {coursePrepaidID: 1});
+  cursor = db.users.find({_id: {$in: classroomUserObjectIds}}, {coursePrepaid: 1, coursePrepaidID: 1});
   while (cursor.hasNext()) {
     doc = cursor.next();
+    userEventEndDateMap[doc._id.valueOf()] = {};
+    userEventEndDateMap[doc._id.valueOf()][eventNameFree] = new Date();
+    if (doc.coursePrepaid) {
+      if (!doc.coursePrepaid.endDate) throw new Error("No endDate for new prepaid " + doc._id.valuOf());
+      userEventEndDateMap[doc._id.valueOf()][eventNamePaid] = new Date(doc.coursePrepaid.endDate);
+      if (!prepaidUsersMap[doc.coursePrepaid._id.valueOf()]) prepaidUsersMap[doc.coursePrepaid._id.valueOf()] = [];
+      prepaidUsersMap[doc.coursePrepaid._id.valueOf()].push(doc._id.valueOf()); 
+      prepaidIDs.push(doc.coursePrepaid._id);
+    }
     if (doc.coursePrepaidID) {
-      userEventMap[doc._id.valueOf()] = eventNamePaid;
+      if (!userEventEndDateMap[doc._id.valueOf()][eventNamePaid]) {
+        userEventEndDateMap[doc._id.valueOf()][eventNamePaid] = new Date();
+      }
       if (!prepaidUsersMap[doc.coursePrepaidID.valueOf()]) prepaidUsersMap[doc.coursePrepaidID.valueOf()] = [];
       prepaidUsersMap[doc.coursePrepaidID.valueOf()].push(doc._id.valueOf()); 
       prepaidIDs.push(doc.coursePrepaidID);
     }
-    else {
-      userEventMap[doc._id.valueOf()] = eventNameFree;
-    }
   }
-  cursor = db.prepaids.find({_id: {$in: prepaidIDs}}, {properties: 1});
+  cursor = db.prepaids.find({_id: {$in: prepaidIDs}}, {endDate: 1, properties: 1});
   while (cursor.hasNext()) {
     doc = cursor.next();
     if (doc.properties && doc.properties.trialRequestID) {
+      var endDate = new Date();
+      if (doc.endDate) {
+        endDate = new Date(doc.endDate);
+      }
+      else if (doc.properties.endDate) {
+        endDate = new Date(doc.properties.endDate);
+      }
       for (var i = 0; i < prepaidUsersMap[doc._id.valueOf()].length; i++) {
-        userEventMap[prepaidUsersMap[doc._id.valueOf()][i]] = eventNameTrial;
+        userEventEndDateMap[prepaidUsersMap[doc._id.valueOf()][i]][eventNameTrial] = endDate;
       }
     }
   }
@@ -121,18 +135,22 @@ function getActiveClassCounts(startDay) {
   var endDate = ISODate(startDay + "T00:00:00.000Z");
   var todayDate = new Date(new Date().toISOString().substring(0, 10));
   var startObj = objectIdWithTimestamp(startDate);
-  var queryParams = {$and: [
+  // Batch size test times: 10k 427005, 5k 361361, 1k 799068, 2k 791521
-    {_id: {$gte: startObj}},
+  var batchSize = 5000;
-    {user: {$in: classroomUserIDs}},
+  for (var j = 0; j < classroomUserIDs.length / batchSize + 1; j++) {
-    {event: 'Started Level'}
+    // log("DEBUG: Fetching classroom events batch " + (j * batchSize) + " " + (j * batchSize + batchSize));
-  ]};
+    var queryParams = {$and: [
-  cursor = logDB['log'].find(queryParams, {user: 1});
+      {_id: {$gte: startObj}},
-  while (cursor.hasNext()) {
+      {user: {$in: classroomUserIDs.slice(j * batchSize, j * batchSize + batchSize)}},
-    doc = cursor.next();
+      {event: 'Started Level'}
-    if (!userPlayedMap[doc.user]) userPlayedMap[doc.user] = [];
+    ]};
-    userPlayedMap[doc.user].push(doc._id.getTimestamp());
+    cursor = analyticsDB['log'].find(queryParams, {user: 1});
+    while (cursor.hasNext()) {
+      doc = cursor.next();
+      if (!userPlayedMap[doc.user]) userPlayedMap[doc.user] = [];
+      userPlayedMap[doc.user].push(doc._id.getTimestamp());
+    }
   }
-  // printjson(userPlayedMap);
 
   log("Calculate number of active members per classroom per day per event type..");
   var classDayTypeMap = {};
@@ -159,7 +177,19 @@ function getActiveClassCounts(startDay) {
         if (userPlayedMap[member]) {
           for (var k = 0; k < userPlayedMap[member].length; k++) {
             if (userPlayedMap[member][k] > startDate && userPlayedMap[member][k] <= endDate) {
-              classDayTypeMap[classroom][endDay][userEventMap[member]]++;
+              if (userEventEndDateMap[member][eventNameTrial] > endDate) {
+                classDayTypeMap[classroom][endDay][eventNameTrial]++;
+              }
+              else if (userEventEndDateMap[member][eventNamePaid] > endDate) {
+                classDayTypeMap[classroom][endDay][eventNamePaid]++;
+              }
+              else if (userEventEndDateMap[member][eventNameFree] > endDate) {
+                classDayTypeMap[classroom][endDay][eventNameFree]++;
+              }
+              else {
+                print("ERROR: no event for " + member);
+                printjson(userEventEndDateMap[member]);
+              }
               break;
             }
           }

scripts/analytics/mongodb/insertAnalyticsActiveUsers.js

@@ -8,6 +8,8 @@
 // Usage:
 // mongo <address>:<port>/<database> <script file> -u <username> -p <password>
 
+// TODO: classroom paid active users before 4/13/16 not correct
+
 try {
   var logDB = new Mongo("localhost").getDB("analytics")
   var scriptStartTime = new Date();
@@ -129,32 +131,47 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
   log("Classroom user count: " + classroomUserObjectIds.length);
 
   // Classrooms free/trial/paid
-  // Paid user: user.coursePrepaidID set means access to paid courses
+  // Paid user: user.coursePrepaid or user.coursePrepaidID set means access to paid courses
   // Trial user: prepaid.properties.trialRequestID means access was via trial
   // Free: not paid, not trial
   log("Finding classroom users free/trial/paid status..");
-  var classroomUserEventMap = {};
+  var classroomUserEventEndDateMap = {};
   var prepaidUsersMap = {};
   var prepaidIDs = [];
-  cursor = db.users.find({_id: {$in: classroomUserObjectIds}}, {coursePrepaidID: 1});
+  cursor = db.users.find({_id: {$in: classroomUserObjectIds}}, {coursePrepaid: 1, coursePrepaidID: 1});
   while (cursor.hasNext()) {
     doc = cursor.next();
+    classroomUserEventEndDateMap[doc._id.valueOf()] = {};
+    classroomUserEventEndDateMap[doc._id.valueOf()]['DAU classroom free'] = new Date();
+    if (doc.coursePrepaid) {
+      if (!doc.coursePrepaid.endDate) throw new Error("No endDate for new prepaid " + doc._id.valuOf());
+      classroomUserEventEndDateMap[doc._id.valueOf()]['DAU classroom paid'] = new Date(doc.coursePrepaid.endDate);
+      if (!prepaidUsersMap[doc.coursePrepaid._id.valueOf()]) prepaidUsersMap[doc.coursePrepaid._id.valueOf()] = [];
+      prepaidUsersMap[doc.coursePrepaid._id.valueOf()].push(doc._id.valueOf()); 
+      prepaidIDs.push(doc.coursePrepaid._id);
+    }
     if (doc.coursePrepaidID) {
-      classroomUserEventMap[doc._id.valueOf()] = 'DAU classroom paid';
+      if (!classroomUserEventEndDateMap[doc._id.valueOf()]['DAU classroom paid']) {
+        classroomUserEventEndDateMap[doc._id.valueOf()]['DAU classroom paid'] = new Date();
+      }
       if (!prepaidUsersMap[doc.coursePrepaidID.valueOf()]) prepaidUsersMap[doc.coursePrepaidID.valueOf()] = [];
       prepaidUsersMap[doc.coursePrepaidID.valueOf()].push(doc._id.valueOf()); 
       prepaidIDs.push(doc.coursePrepaidID);
     }
-    else {
-      classroomUserEventMap[doc._id.valueOf()] = 'DAU classroom free';
-    }
   }
   cursor = db.prepaids.find({_id: {$in: prepaidIDs}}, {properties: 1});
   while (cursor.hasNext()) {
     doc = cursor.next();
     if (doc.properties && doc.properties.trialRequestID) {
+      var endDate = new Date();
+      if (doc.endDate) {
+        endDate = new Date(doc.endDate);
+      }
+      else if (doc.properties.endDate) {
+        endDate = new Date(doc.properties.endDate);
+      }
       for (var i = 0; i < prepaidUsersMap[doc._id.valueOf()].length; i++) {
-        classroomUserEventMap[prepaidUsersMap[doc._id.valueOf()][i]] = 'DAU classroom trial';
+        classroomUserEventEndDateMap[prepaidUsersMap[doc._id.valueOf()][i]]['DAU classroom trial'] = endDate;
       }
     }
   }
@@ -199,7 +216,22 @@ function getActiveUserCounts(startDay, endDay, activeUserEvents) {
   var userDayEventMap = {}
   for (day in dayUserActiveMap) {
     for (var user in dayUserActiveMap[day]) {
-      var event = classroomUserEventMap[user] || (dayCampaignUserPaidMap[day] && dayCampaignUserPaidMap[day][user] ? 'DAU campaign paid' : 'DAU campaign free');
+      var event = null;
+      var endDate = new Date(day + "T00:00:00.000Z");
+      if (classroomUserEventEndDateMap[user]) {
+        if (classroomUserEventEndDateMap[user]['DAU classroom trial'] > endDate) {
+          event = 'DAU classroom trial';
+        }
+        else if (classroomUserEventEndDateMap[user]['DAU classroom paid'] > endDate) {
+          event = 'DAU classroom paid';
+        }
+        else if (classroomUserEventEndDateMap[user]['DAU classroom free'] > endDate) {
+          event = 'DAU classroom free';
+        }
+      }
+      if (!event) {
+        event = dayCampaignUserPaidMap[day] && dayCampaignUserPaidMap[day][user] ? 'DAU campaign paid' : 'DAU campaign free';
+      }
       dailyEventNames[event] = true;
       if (!activeUsersCounts[day]) activeUsersCounts[day] = {};
       if (!activeUsersCounts[day][event]) activeUsersCounts[day][event] = 0;