Analytics data revamp

Add analytics per-day aggregation collection.
Add analytics strings collection.
Add per-day aggregation mongo insertion script.
Update campaign editor to use aggregation collection.
Update queries to use _id instead of created field.

app/core/utils.coffee

@@ -131,7 +131,7 @@ module.exports.kindaEqual = compare = (l, r) ->
   else
     return false
 
-# Return UTC string "YYYY-MM-DD" for today + offset
+# Return UTC string "YYYYMMDD" for today + offset
 module.exports.getUTCDay = (offset=0) ->
   day = new Date()
   day.setDate(day.getUTCDate() + offset)
@@ -140,7 +140,7 @@ module.exports.getUTCDay = (offset=0) ->
   partMonth = "0" + partMonth if partMonth < 10
   partDay = day.getUTCDate()
   partDay = "0" + partDay if partDay < 10
-  "#{partYear}-#{partMonth}-#{partDay}"
+  "#{partYear}#{partMonth}#{partDay}"
 
 # Fast, basic way to replace text in an element when you don't need much.
 # http://stackoverflow.com/a/4962398/540620

app/schemas/models/analytics_perday.coffee

@@ -0,0 +1,17 @@
+c = require './../schemas'
+
+AnalyticsPerDaySchema = c.object {
+  title: 'Analytics per-day data'
+  description: 'Analytics data aggregated into per-day chunks.'
+}
+
+_.extend AnalyticsPerDaySchema.properties,
+  d: {type: 'string'}   # yyyymmdd day, e.g. '20150123'
+  l: {type: 'integer'}  # level (analytics ID from analytics.strings)
+  f: {type: 'integer'}  # filter (analytics ID from analytics.strings)
+  fv: {type: 'integer'} # filter value (analytics ID from analytics.strings)
+  c: {type: 'integer'}  # count
+
+c.extendBasicProperties AnalyticsPerDaySchema, 'analytics.perday'
+
+module.exports = AnalyticsPerDaySchema

app/schemas/models/analytics_string.coffee

@@ -0,0 +1,13 @@
+c = require './../schemas'
+
+AnalyticsStringSchema = c.object {
+  title: 'Analytics String'
+  description: 'Maps strings to number IDs for improved performance.'
+}
+
+_.extend AnalyticsStringSchema.properties,
+  v: {type: 'string'} # value
+
+c.extendBasicProperties AnalyticsStringSchema, 'analytics.string'
+
+module.exports = AnalyticsStringSchema

app/templates/editor/campaign/campaign-editor-view.jade

@@ -40,7 +40,7 @@ block outer_content
     #right-column
       #campaign-view
       #campaign-level-view.hidden
-      if campaignDropOffs
+      if campaignCompletions
         button.btn.btn-default#analytics-button(title="Analytics",  data-toggle="modal" data-target="#analytics-modal") Analytics
         .modal.fade#analytics-modal(tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true")
           .modal-dialog
@@ -50,33 +50,25 @@ block outer_content
                   span(aria-hidden="true") ×
                 h4.modal-title Analytics
               .modal-body
-                if campaignDropOffs.startDay
-                  if campaignDropOffs.endDay
-                    div #{campaignDropOffs.startDay} to #{campaignDropOffs.endDay}
+                if campaignCompletions.startDay
+                  if campaignCompletions.endDay
+                    div #{campaignCompletions.startDay} to #{campaignCompletions.endDay}
                   else
-                    div #{campaignDropOffs.startDay} to today
+                    div #{campaignCompletions.startDay} to yesterday
                 table.table.table-bordered.table-condensed.table-hover(style='font-size:10pt')
                   thead
                     tr
                       td Level
                       td Started
-                      td Dropped
-                      td Drop %
                       td Finished
-                      td Dropped
-                      td Drop %
                       td Completion %
                   tbody
-                    - for (var i = 0; i < campaignDropOffs.levels.length; i++)
+                    - for (var i = 0; i < campaignCompletions.levels.length; i++)
                       tr
-                        td= campaignDropOffs.levels[i].level
-                        td= campaignDropOffs.levels[i].started
-                        td= campaignDropOffs.levels[i].startDropped
-                        td= campaignDropOffs.levels[i].startDropRate
-                        td= campaignDropOffs.levels[i].finished
-                        td= campaignDropOffs.levels[i].finishDropped
-                        td= campaignDropOffs.levels[i].finishDropRate
-                        td= campaignDropOffs.levels[i].completionRate
+                        td= campaignCompletions.levels[i].level
+                        td= campaignCompletions.levels[i].started
+                        td= campaignCompletions.levels[i].finished
+                        td= campaignCompletions.levels[i].completionRate
       else
         button.btn.btn-default.disabled#analytics-button Analytics Loading...
 

app/views/editor/campaign/CampaignEditorView.coffee

@@ -47,7 +47,7 @@ module.exports = class CampaignEditorView extends RootView
     @listenToOnce @levels, 'sync', @onFundamentalLoaded
     @listenToOnce @achievements, 'sync', @onFundamentalLoaded
 
-    #_.delay @getCampaignCompletions, 1000  # Roughly never finishes loading, nearly kills server.
+    _.delay @getCampaignCompletions, 500
 
   loadThangTypeNames: ->
     # Load the names of the ThangTypes that this level's Treema nodes might want to display.
@@ -132,7 +132,7 @@ module.exports = class CampaignEditorView extends RootView
   getRenderData: ->
     c = super()
     c.campaign = @campaign
-    c.campaignDropOffs = @campaignDropOffs
+    c.campaignCompletions = @campaignCompletions
     c
 
   onClickSaveButton: ->
@@ -241,26 +241,24 @@ module.exports = class CampaignEditorView extends RootView
         @toSave.add achievement
 
   getCampaignCompletions: =>
-    # Fetch last 7 days of campaign drop-off rates
+    # Fetch last 14 days of campaign drop-off rates
 
-    startDay = utils.getUTCDay -6
+    startDay = utils.getUTCDay -14
+    endDay = utils.getUTCDay -1
 
     success = (data) =>
       return if @destroyed
-      # API returns all the campaign data currently
-      @campaignDropOffs = data[@campaignHandle]
       mapFn = (item) ->
-        item.startDropRate = (item.startDropped / item.started * 100).toFixed(2)
-        item.finishDropRate = (item.finishDropped / item.finished * 100).toFixed(2)
         item.completionRate = (item.finished / item.started * 100).toFixed(2)
         item
-      @campaignDropOffs.levels = _.map @campaignDropOffs.levels, mapFn, @
-      @campaignDropOffs.startDay = startDay
+      @campaignCompletions = levels:  _.map data, mapFn, @
+      @campaignCompletions.startDay = "#{startDay[0..3]}-#{startDay[4..5]}-#{startDay[6..7]}"
+      @campaignCompletions.endDay = "#{endDay[0..3]}-#{endDay[4..5]}-#{endDay[6..7]}"
       @render()
 
     # TODO: Why do we need this url dash?
     request = @supermodel.addRequestResource 'campaign_completions', {
-      url: '/db/analytics_log_event/-/campaign_completions'
+      url: '/db/analytics_perday/-/campaign_completions'
       data: {startDay: startDay, slug: @campaignHandle}
       method: 'POST'
       success: success

app/views/editor/campaign/CampaignLevelView.coffee

@@ -49,6 +49,7 @@ module.exports = class CampaignLevelView extends CocoView
   getCommonLevelProblems: ->
     # Fetch last 30 days of common level problems
     startDay = utils.getUTCDay -29
+    startDay = "#{startDay[0..3]}-#{startDay[4..5]}-#{startDay[6..7]}"
 
     success = (data) =>
       return if @destroyed
@@ -66,7 +67,7 @@ module.exports = class CampaignLevelView extends CocoView
     request.load()
 
   getLevelCompletions: ->
-    # Fetch last 7 days of level completion counts
+    # Fetch last 14 days of level completion counts
     success = (data) =>
       return if @destroyed
       data.sort (a, b) -> if a.created < b.created then 1 else -1
@@ -76,11 +77,11 @@ module.exports = class CampaignLevelView extends CocoView
       @levelCompletions = _.map data, mapFn, @
       @render()
 
-    startDay = utils.getUTCDay -13
+    startDay = utils.getUTCDay -14
     
     # TODO: Why do we need this url dash?
     request = @supermodel.addRequestResource 'level_completions', {
-      url: '/db/analytics_log_event/-/level_completions'
+      url: '/db/analytics_perday/-/level_completions'
       data: {startDay: startDay, slug: @levelSlug}
       method: 'POST'
       success: success
@@ -88,13 +89,14 @@ module.exports = class CampaignLevelView extends CocoView
     request.load()
 
   getLevelPlaytimes: ->
-    # Fetch last 7 days of level average playtimes
+    # Fetch last 14 days of level average playtimes
     success = (data) =>
       return if @destroyed
       @levelPlaytimes = data.sort (a, b) -> if a.created < b.created then 1 else -1
       @render()
 
     startDay = utils.getUTCDay -13
+    startDay = "#{startDay[0..3]}-#{startDay[4..5]}-#{startDay[6..7]}"
 
     # TODO: Why do we need this url dash?
     request = @supermodel.addRequestResource 'playtime_averages', {

scripts/analytics/mongodb/queries/campaignRates.js

@@ -18,17 +18,27 @@ today = today.toISOString().substr(0, 10);
 print("Today is " + today);
 
 var todayMinus6 = new Date();
-todayMinus6.setDate(todayMinus6.getUTCDate() - 6);
+todayMinus6.setUTCDate(todayMinus6.getUTCDate() - 6);
 var startDate = todayMinus6.toISOString().substr(0, 10) + "T00:00:00.000Z";
 // startDate = "2014-12-31T00:00:00.000Z";
 print("Start date is " + startDate)
 // var endDate = "2015-01-06T00:00:00.000Z";
 // print("End date is " + endDate)
 
+function objectIdWithTimestamp(timestamp)
+{
+  // Convert string date to Date object (otherwise assume timestamp is a date)
+  if (typeof(timestamp) == 'string') timestamp = new Date(timestamp);
+  // Convert date object to hex seconds since Unix epoch
+  var hexSeconds = Math.floor(timestamp/1000).toString(16);
+  // Create an ObjectId with that hex timestamp
+  var constructedObjectId = ObjectId(hexSeconds + "0000000000000000");
+  return constructedObjectId
+}
+
 var cursor = db['analytics.log.events'].find({
   $and: [
-    {"created": { $gte: ISODate(startDate)}},
-    // {"created": { $lt: ISODate(endDate)}},
+    {_id: {$gte: objectIdWithTimestamp(ISODate(startDate))}},
     {$or: [ {"event" : 'Started Level'}, {"event" : 'Saw Victory'}]}
     ]
 });

scripts/analytics/mongodb/queries/insertPerDayAnalytics.js

@@ -0,0 +1,207 @@
+// Insert per-day analytics into analytics.perdays collection
+
+// Usage:
+// mongo <address>:<port>/<database> <script file> -u <username> -p <password>
+
+// Completion rates (funnels) are calculated like Mixpanel
+// For a given date range, start count is the number of first steps (e.g. started a level)
+// Finish count for the same start date is how many unique users finished the remaining steps in the following ~30 days
+// https://mixpanel.com/help/questions/articles/how-are-funnels-calculated
+
+// TODO: Why are Mixpanel level finish events significantly lower?
+// TODO: dungeons-of-kithgard completion rate is 62% vs. 77%
+// TODO: Similar start events, finish events off by 20% (5334 vs 6486)
+// TODO: Are Mixpanel rates accounting for finishing steps likely to be completed in the future?
+// TODO: Use Mixpanel export API to investigate
+
+// TODO: Output documents updated/inserted
+
+var scriptStartTime = new Date();
+var analyticsStringCache = {};
+
+function log(str) {
+  print(new Date().toISOString() + " " + str);
+}
+
+function objectIdWithTimestamp(timestamp) {
+  // Convert string date to Date object (otherwise assume timestamp is a date)
+  if (typeof(timestamp) == 'string') timestamp = new Date(timestamp);
+  // Convert date object to hex seconds since Unix epoch
+  var hexSeconds = Math.floor(timestamp/1000).toString(16);
+  // Create an ObjectId with that hex timestamp
+  var constructedObjectId = ObjectId(hexSeconds + "0000000000000000");
+  return constructedObjectId
+}
+
+function getAnalyticsString(str) {
+  if (analyticsStringCache[str]) return analyticsStringCache[str];
+
+  // Find existing string
+  var doc = db['analytics.strings'].findOne({v: str});
+  if (doc) {
+    analyticsStringCache[str] = doc._id;
+    return analyticsStringCache[str];
+  }
+
+  // Insert string
+  // http://docs.mongodb.org/manual/tutorial/create-an-auto-incrementing-field/#auto-increment-optimistic-loop
+  doc = {v: str};
+  while (true) {
+    var cursor = db['analytics.strings'].find({}, {_id: 1}).sort({_id: -1}).limit(1);
+    var seq = cursor.hasNext() ? cursor.next()._id + 1 : 1;
+    doc._id = seq;
+    var results = db['analytics.strings'].insert(doc);
+    if (results.hasWriteError()) {
+      if ( results.writeError.code == 11000 /* dup key */ ) continue;
+      else throw new Error("ERROR: Unexpected error inserting data: " + tojson(results));
+    }
+    break;
+  }
+
+  // Find new string entry
+  doc = db['analytics.strings'].findOne({v: str});
+  if (doc) {
+    analyticsStringCache[str] = doc._id;
+    return analyticsStringCache[str];
+  }
+  throw new Error("ERROR: Did not find analytics.strings insert for: " + str);
+}
+
+function getLevelFunnelData(startDay, eventFunnel) {
+  if (!startDay || !eventFunnel || eventFunnel.length === 0) return {};
+
+  var startObj = objectIdWithTimestamp(ISODate(startDay + "T00:00:00.000Z"));
+  var queryParams = {$and: [{_id: {$gte: startObj}},{"event": {$in: eventFunnel}}]};
+  var cursor = db['analytics.log.events'].find(queryParams);
+
+  // Map ordering: level, user, event, created
+  var userDataMap = {};
+  while (cursor.hasNext()) {
+    var doc = cursor.next();
+    var created = doc._id.getTimestamp().toISOString().substring(0, 10);
+    var event = doc.event;
+    var properties = doc.properties;
+    var user = doc.user;
+    var level;
+
+    // TODO: Switch to properties.levelID for 'Saw Victory'
+    if (event === 'Saw Victory' && properties.level) level = properties.level.toLowerCase().replace(/ /g, '-');
+    else if (properties.levelID) level = properties.levelID
+    else continue
+
+    if (!userDataMap[level]) userDataMap[level] = {};
+    if (!userDataMap[level][user]) userDataMap[level][user] = {};
+    if (!userDataMap[level][user][event] || userDataMap[level][user][event].localeCompare(created) > 0) {
+      // if (userDataMap[level][user][event]) log("Found earlier date " + level + " " + event + " " + user + " " + userDataMap[level][user][event] + " " + created);
+      userDataMap[level][user][event] = created;
+    }
+  }
+
+  // Data: level, created, event
+  var levelFunnelData = {};
+  for (level in userDataMap) {
+    for (user in userDataMap[level]) {
+
+      // Find first event date
+      var funnelStartDay = null;
+      for (event in userDataMap[level][user]) {
+        var created = userDataMap[level][user][event];
+        if (!levelFunnelData[level]) levelFunnelData[level] = {};
+        if (!levelFunnelData[level][created]) levelFunnelData[level][created] = {};
+        if (!levelFunnelData[level][created][event]) levelFunnelData[level][created][event] = 0;
+        if (eventFunnel[0] === event) {
+          // First event gets attributed to current date
+          levelFunnelData[level][created][event]++;
+          funnelStartDay = created;
+          break;
+        }
+      }
+
+      if (funnelStartDay) {
+        // Add remaining funnel steps/events to first step's date
+        for (event in userDataMap[level][user]) {
+          if (!levelFunnelData[level][funnelStartDay][event]) levelFunnelData[level][funnelStartDay][event] = 0;
+          if (eventFunnel[0] != event) levelFunnelData[level][funnelStartDay][event]++;
+        }
+        // Zero remaining funnel events
+        for (var i = 1; i < eventFunnel.length; i++) {
+          var event = eventFunnel[i];
+          if (!levelFunnelData[level][funnelStartDay][event]) levelFunnelData[level][funnelStartDay][event] = 0;
+        }
+      }
+      // Else no start event in this date range
+    }
+  }
+  return levelFunnelData;
+}
+
+function insertEventCount(event, level, day, count) {
+  // analytics.perdays schema in server/analytics/AnalyticsPeryDay.coffee
+  day = day.replace(/-/g, '');
+
+  var eventID = getAnalyticsString(event);
+  var levelID = getAnalyticsString(level);
+  var filterID = getAnalyticsString('all');
+
+  var startObj = objectIdWithTimestamp(ISODate(startDay + "T00:00:00.000Z"));
+  var queryParams = {$and: [{d: day}, {e: eventID}, {l: levelID}, {f: filterID}]};
+  var doc = db['analytics.perdays'].findOne(queryParams);
+  if (doc && doc.c === count) return;
+
+  if (doc && doc.c !== count) {
+    // Update existing count, assume new one is more accurate
+    // log("Updating count in db for " + day + " " + event + " " + level + " " + doc.c + " => " + count);
+    var results = db['analytics.perdays'].update(queryParams, {$set: {c: count}});
+    if (results.nMatched !== 1 && results.nModified !== 1) {
+      log("ERROR: update count failed");
+      printjson(results);
+    }
+  }
+  else {
+    var insertDoc = {d: day, e: eventID, l: levelID, f: filterID, c: count};
+    var results = db['analytics.perdays'].insert(insertDoc);
+    if (results.nInserted !== 1) {
+      log("ERROR: insert event failed");
+      printjson(results);
+      printjson(insertDoc);
+    }
+    // else {
+    //   log("Added " + day + " " + event + " " + count + " " + level);
+    // }
+  }
+}
+
+try {
+  // Look at last 30 days, same as Mixpanel
+  var numDays = 30;
+  
+  var startDay = new Date();
+  today = startDay.toISOString().substr(0, 10);
+  startDay.setUTCDate(startDay.getUTCDate() - numDays);
+  startDay = startDay.toISOString().substr(0, 10);
+
+  var levelCompletionFunnel = ['Started Level', 'Saw Victory'];
+
+  log("Today is " + today);
+  log("Start day is " + startDay);
+  log("Funnel events are " + levelCompletionFunnel);
+  
+  log("Getting level completion data...");
+  var levelCompletionData = getLevelFunnelData(startDay, levelCompletionFunnel);
+  
+  log("Inserting aggregated level completion data...");
+  for (level in levelCompletionData) {
+    for (created in levelCompletionData[level]) {
+      if (today === created) continue; // Never save data for today because it's incomplete
+      for (event in levelCompletionData[level][created]) {
+        insertEventCount(event, level, created, levelCompletionData[level][created][event]);
+      }
+    }
+  }
+} 
+catch(err) {
+  log("ERROR: " + err);
+  printjson(err);
+}
+
+log("Script runtime: " + (new Date() - scriptStartTime));

scripts/analytics/mongodb/queries/levelRates.js

@@ -21,19 +21,28 @@ today = today.toISOString().substr(0, 10);
 print("Today is " + today);
 
 var todayMinus6 = new Date();
-todayMinus6.setDate(todayMinus6.getUTCDate() - 6);
+todayMinus6.setUTCDate(todayMinus6.getUTCDate() - 6);
 var startDate = todayMinus6.toISOString().substr(0, 10) + "T00:00:00.000Z";
 print("Start date is " + startDate)
 // startDate = "2015-01-02T00:00:00.000Z";
 // var endDate = "2015-01-09T00:00:00.000Z";
 
+function objectIdWithTimestamp(timestamp)
+{
+  // Convert string date to Date object (otherwise assume timestamp is a date)
+  if (typeof(timestamp) == 'string') timestamp = new Date(timestamp);
+  // Convert date object to hex seconds since Unix epoch
+  var hexSeconds = Math.floor(timestamp/1000).toString(16);
+  // Create an ObjectId with that hex timestamp
+  var constructedObjectId = ObjectId(hexSeconds + "0000000000000000");
+  return constructedObjectId
+}
+
 function getCompletionRates() {
   print("Getting completion rates...");
   var queryParams = {
     $and: [
-    {"created": { $gte: ISODate(startDate)}},
-    // {"created": { $lt: ISODate(endDate)}},
-    // {$or: [ {"properties.level": {$exists: true}}, {"properties.levelID": {$exists: true}}]},
+    {_id: {$gte: objectIdWithTimestamp(ISODate(startDate))}},
     {$or: [ {"event" : 'Started Level'}, {"event" : 'Saw Victory'}]}
     ]
   };

server/analytics/AnalyticsPerDay.coffee

@@ -0,0 +1,12 @@
+mongoose = require 'mongoose'
+
+AnalyticsPerDaySchema = new mongoose.Schema({
+  e: {type: Number}  # event (analytics string ID from analytics.strings)
+  l: {type: Number}  # level (analytics string ID from analytics.strings)
+  f: {type: Number}  # filter (analytics string ID from analytics.strings)
+  fv: {type: Number} # filter value (analytics string ID from analytics.strings)
+  c: {type: Number}  # count
+}, {strict: false})
+
+# TODO: Why can't we query against a collection with caps, like 'analytics.perDay'?
+module.exports = AnalyticsPerDay = mongoose.model('analytics.perday', AnalyticsPerDaySchema)

server/analytics/AnalyticsString.coffee

@@ -0,0 +1,12 @@
+mongoose = require 'mongoose'
+
+# Auto-incrementing number _id
+# http://docs.mongodb.org/manual/tutorial/create-an-auto-incrementing-field/#auto-increment-optimistic-loop
+# TODO: Why strict:false?
+
+AnalyticsStringSchema = new mongoose.Schema({
+  _id: {type: Number}
+  v: {type: String}
+}, {strict: false})
+
+module.exports =  AnalyticsString = mongoose.model('analytics.string', AnalyticsStringSchema)

server/analytics/analytics_log_event_handler.coffee

@@ -3,6 +3,7 @@ Campaign = require '../campaigns/Campaign'
 Level = require '../levels/Level'
 Handler = require '../commons/Handler'
 log = require 'winston'
+utils = require '../lib/utils'
 
 class AnalyticsLogEventHandler extends Handler
   modelClass: AnalyticsLogEvent
@@ -60,8 +61,8 @@ class AnalyticsLogEventHandler extends Handler
     queryParams = {$and: [
       {$or: [{"event" : 'Started Level'}, {"event" : 'Saw Victory'}]}
     ]}
-    queryParams["$and"].push created: {$gte: new Date(startDay + "T00:00:00.000Z")} if startDay?
-    queryParams["$and"].push created: {$lt: new Date(endDay + "T00:00:00.000Z")} if endDay?
+    queryParams["$and"].push _id: {$gte: utils.objectIdFromTimestamp(startDay + "T00:00:00.000Z")} if startDay?
+    queryParams["$and"].push _id: {$lt: utils.objectIdFromTimestamp(endDay + "T00:00:00.000Z")} if endDay?
 
     # Query stream is better for large results
     # http://mongoosejs.com/docs/api.html#query_Query-stream
@@ -221,8 +222,8 @@ class AnalyticsLogEventHandler extends Handler
       userProgression = {}
 
       queryParams = {$and: [{$or: [ {"event" : 'Started Level'}, {"event" : 'Saw Victory'}]}]}
-      queryParams["$and"].push created: {$gte: new Date(startDay + "T00:00:00.000Z")} if startDay?
-      queryParams["$and"].push created: {$lt: new Date(endDay + "T00:00:00.000Z")} if endDay?
+      queryParams["$and"].push _id: {$gte: utils.objectIdFromTimestamp(startDay + "T00:00:00.000Z")} if startDay?
+      queryParams["$and"].push _id: {$lt: utils.objectIdFromTimestamp(endDay + "T00:00:00.000Z")} if endDay?
 
       # Query stream is better for large results
       # http://mongoosejs.com/docs/api.html#query_Query-stream

server/analytics/analytics_perday_handler.coffee

@@ -0,0 +1,193 @@
+AnalyticsPerDay = require './AnalyticsPerDay'
+AnalyticsString = require './AnalyticsString'
+Campaign = require '../campaigns/Campaign'
+Level = require '../levels/Level'
+Handler = require '../commons/Handler'
+log = require 'winston'
+
+class AnalyticsPerDayHandler extends Handler
+  modelClass: AnalyticsPerDay
+  jsonSchema: require '../../app/schemas/models/analytics_perday'
+
+  hasAccess: (req) ->
+    req.method in ['GET'] or req.user?.isAdmin()
+
+  getByRelationship: (req, res, args...) ->
+    return @getCampaignCompletionsBySlug(req, res) if args[1] is 'campaign_completions'
+    return @getLevelCompletionsBySlug(req, res) if args[1] is 'level_completions'
+    super(arguments...)
+
+  getCampaignCompletionsBySlug: (req, res) ->
+    # Send back an array of level starts and finishes
+    # Parameters:
+    # slug - campaign slug
+    # startDay - Inclusive, optional, YYYYMMDD e.g. '20141214'
+    # endDay - Exclusive, optional, YYYYMMDD e.g. '20141216'
+
+    campaignSlug = req.query.slug or req.body.slug
+    startDay = req.query.startDay or req.body.startDay
+    endDay = req.query.endDay or req.body.endDay
+
+    # log.warn "campaign_completions campaignSlug='#{campaignSlug}' startDay=#{startDay} endDay=#{endDay}"
+
+    return @sendSuccess res, [] unless campaignSlug?
+
+    # Cache results in app server memory for 1 day
+    @campaignCompletionsCache ?= {}
+    @campaignCompletionsCachedSince ?= new Date()
+    if (new Date()) - @campaignCompletionsCachedSince > 86400 * 1000
+      @campaignCompletionsCache = {}
+      @campaignCompletionsCachedSince = new Date()
+    cacheKey = campaignSlug
+    cacheKey += 's' + startDay if startDay?
+    cacheKey += 'e' + endDay if endDay?
+    return @sendSuccess res, campaignDropOffs if campaignDropOffs = @campaignCompletionsCache[cacheKey]
+
+    getCompletions = (orderedLevelSlugs, levelStringIDSlugMap) =>
+      # 3. Send back an array of level starts and finishes
+      # Input:
+      # orderedLevelSlugs - Ordered list of level slugs, used for sorting results
+      # levelStringIDSlugMap - Maps level string IDs to level slugs
+
+      campaignLevelIDs = Object.keys(levelStringIDSlugMap)
+
+      AnalyticsString.find({v: {$in: ['Started Level', 'Saw Victory', 'all']}}).exec (err, documents) =>
+        if err? then return @sendDatabaseError res, err
+
+        for doc in documents
+          startEventID = doc._id if doc.v is 'Started Level'
+          finishEventID = doc._id if doc.v is 'Saw Victory'
+          filterEventID =  doc._id if doc.v is 'all'
+        return @sendSuccess res, [] unless startEventID? and finishEventID? and filterEventID?
+
+        queryParams = {$and: [
+          {$or: [{e: startEventID}, {e: finishEventID}]},
+          {f: filterEventID},
+          {l: {$in: campaignLevelIDs}}
+        ]}
+        queryParams["$and"].push {d: {$gte: startDay}} if startDay?
+        queryParams["$and"].push {d: {$lt: endDay}} if endDay?
+        AnalyticsPerDay.find(queryParams).exec (err, documents) =>
+          if err? then return @sendDatabaseError res, err
+
+          levelEventCounts = {}
+          for doc in documents
+            levelEventCounts[doc.l] ?= {}
+            levelEventCounts[doc.l][doc.e] ?= 0
+            levelEventCounts[doc.l][doc.e] += doc.c
+
+          completions = []
+          for levelID of levelEventCounts
+            completions.push
+              level: levelStringIDSlugMap[levelID]
+              started: levelEventCounts[levelID][startEventID]
+              finished: levelEventCounts[levelID][finishEventID]
+          completions.sort (a, b) -> orderedLevelSlugs.indexOf(a.level) - orderedLevelSlugs.indexOf(b.level)
+
+          @campaignCompletionsCache[cacheKey] = completions
+          @sendSuccess res, completions
+
+    getLevelData = (campaignLevels) =>
+      # 2. Get ordered level slugs and string ID to level slug mappping
+      # Input:
+      # campaignLevels - array of Level IDs
+
+      queryParams = {original: {$in: campaignLevels}, "version.isLatestMajor": true, "version.isLatestMinor": true}
+      Level.find(queryParams).exec (err, documents) =>
+        if err? then return @sendDatabaseError res, err
+
+        # Save original level ID and slug in array for sorting
+        campaignOriginalSlugs = []
+        for doc in documents
+          campaignOriginalSlugs.push
+            slug: doc.get('name').toLowerCase().replace new RegExp(' ', 'g'), '-'
+            original: doc.get('original').toString()
+
+        # Sort slugs against original levels from campaign
+        campaignOriginalSlugs.sort (a, b) ->
+          if campaignLevels.indexOf(a.original) < campaignLevels.indexOf(b.original) then -1 else 1
+
+        # Lookup analytics string IDs for level slugs
+        orderedLevelSlugs = []
+        orderedLevelSlugs.push item.slug for item in campaignOriginalSlugs
+        AnalyticsString.find({v: {$in: orderedLevelSlugs}}).exec (err, documents) =>
+          if err? then return @sendDatabaseError res, err
+
+          levelStringIDSlugMap = {}
+          levelStringIDSlugMap[doc._id] = doc.v for doc in documents
+          getCompletions orderedLevelSlugs, levelStringIDSlugMap
+
+    # 1. Get campaign levels 
+    Campaign.find({slug: campaignSlug}).exec (err, documents) =>
+      if err? then return @sendDatabaseError res, err
+      campaignLevels = []
+      campaignLevels.push level for level of doc.get('levels') for doc in documents
+      getLevelData campaignLevels
+
+  getLevelCompletionsBySlug: (req, res) ->
+    # Returns an array of per-day starts and finishes for given level
+    # Parameters:
+    # slug - level slug
+    # startDay - Inclusive, optional, YYYYMMDD e.g. '20141214'
+    # endDay - Exclusive, optional, YYYYMMDD e.g. '20141216'
+
+    # TODO: Code is similar to getCampaignCompletionsBySlug
+
+    levelSlug = req.query.slug or req.body.slug
+    startDay = req.query.startDay or req.body.startDay
+    endDay = req.query.endDay or req.body.endDay
+
+    return @sendSuccess res, [] unless levelSlug?
+
+    # log.warn "level_completions levelSlug='#{levelSlug}' startDay=#{startDay} endDay=#{endDay}"
+
+    # Cache results in app server memory for 1 day
+    @levelCompletionsCache ?= {}
+    @levelCompletionsCachedSince ?= new Date()
+    if (new Date()) - @levelCompletionsCachedSince > 86400 * 1000
+      @levelCompletionsCache = {}
+      @levelCompletionsCachedSince = new Date()
+    cacheKey = levelSlug
+    cacheKey += 's' + startDay if startDay?
+    cacheKey += 'e' + endDay if endDay?
+    return @sendSuccess res, levelCompletions if levelCompletions = @levelCompletionsCache[cacheKey]
+
+    AnalyticsString.find({v: {$in: ['Started Level', 'Saw Victory', 'all', levelSlug]}}).exec (err, documents) =>
+      if err? then return @sendDatabaseError res, err
+
+      for doc in documents
+        startEventID = doc._id if doc.v is 'Started Level'
+        finishEventID = doc._id if doc.v is 'Saw Victory'
+        filterEventID =  doc._id if doc.v is 'all'
+        levelID = doc._id if doc.v is levelSlug
+      return @sendSuccess res, [] unless startEventID? and finishEventID? and filterEventID? and levelID?
+
+      queryParams = {$and: [{$or: [{e: startEventID}, {e: finishEventID}]},{f: filterEventID},{l: levelID}]}
+      queryParams["$and"].push {d: {$gte: startDay}} if startDay?
+      queryParams["$and"].push {d: {$lt: endDay}} if endDay?
+      AnalyticsPerDay.find(queryParams).exec (err, documents) =>
+        if err? then return @sendDatabaseError res, err
+
+        dayEventCounts = {}
+        for doc in documents
+          day = doc.get('d')
+          eventID = doc.get('e')
+          count = doc.get('c')
+          dayEventCounts[day] ?= {}
+          dayEventCounts[day][eventID] = count
+
+        completions = []
+        for day of dayEventCounts
+          for eventID of dayEventCounts[day]
+            eventID = parseInt eventID
+            started = dayEventCounts[day][eventID] if eventID is startEventID
+            finished = dayEventCounts[day][eventID] if eventID is finishEventID
+          completions.push
+            created: day
+            started: started
+            finished: finished
+
+        @levelCompletionsCache[cacheKey] = completions
+        @sendSuccess res, completions
+
+module.exports = new AnalyticsPerDayHandler()

server/analytics/analytics_string_handler.coffee

@@ -0,0 +1,9 @@
+AnalyticsString = require './AnalyticsString'
+Handler = require '../commons/Handler'
+
+class AnalyticsStringHandler extends Handler
+  modelClass: AnalyticsString
+  jsonSchema: require '../../app/schemas/models/analytics_string'
+  hasAccess: (req) -> req.method in ['GET'] or req.user?.isAdmin()
+
+module.exports = new AnalyticsStringHandler()

server/commons/mapping.coffee

@@ -1,5 +1,7 @@
 module.exports.handlers =
   'analytics_log_event': 'analytics/analytics_log_event_handler'
+  'analytics_perday': 'analytics/analytics_perday_handler'
+  'analytics_string': 'analytics/analytics_string_handler'
   # TODO: Disabling this until we know why our app servers CPU grows out of control.
   # 'analytics_users_active': 'analytics/analytics_users_active_handler'
   'article': 'articles/article_handler'

server/levels/level_handler.coffee

@@ -6,6 +6,7 @@ Feedback = require './feedbacks/LevelFeedback'
 Handler = require '../commons/Handler'
 mongoose = require 'mongoose'
 async = require 'async'
+utils = require '../lib/utils'
 
 LevelHandler = class LevelHandler extends Handler
   modelClass: Level
@@ -375,9 +376,9 @@ LevelHandler = class LevelHandler extends Handler
 
     # Build query
     match = {$match: {$and: [{"state.complete": true}, {"playtime": {$gt: 0}}, {levelID: {$in: levelSlugs}}]}}
-    match["$match"]["$and"].push created: {$gte: new Date(startDay + "T00:00:00.000Z")} if startDay?
-    match["$match"]["$and"].push created: {$lt: new Date(endDay + "T00:00:00.000Z")} if endDay?
-    project = {"$project": {"_id": 0, "levelID": 1, "playtime": 1, "created": {"$concat": [{"$substr":  ["$created", 0, 4]}, "-", {"$substr":  ["$created", 5, 2]}, "-", {"$substr" :  ["$created", 8, 2]}]}}}
+    match["$match"]["$and"].push _id: {$gte: utils.objectIdFromTimestamp(startDay + "T00:00:00.000Z")} if startDay?
+    match["$match"]["$and"].push _id: {$lt: utils.objectIdFromTimestamp(endDay + "T00:00:00.000Z")} if endDay?
+    project = {"$project": {"_id": 0, "levelID": 1, "playtime": 1, "created": {"$concat": [{"$substr":  ["$created", 0, 10]}]}}}
     group = {"$group": {"_id": {"created": "$created", "level": "$levelID"}, "average": {"$avg": "$playtime"}}}
     query = Session.aggregate match, project, group
 

server/lib/utils.coffee

@@ -1,3 +1,15 @@
+mongoose = require 'mongoose'
 
 module.exports =
   isID: (id) -> _.isString(id) and id.length is 24 and id.match(/[a-f0-9]/gi)?.length is 24
+  objectIdFromTimestamp: (timestamp) ->
+    # mongoDB ObjectId contains creation date in first 4 bytes
+    # So, it can be used instead of a redundant created field
+    # http://docs.mongodb.org/manual/reference/object-id/
+    # http://stackoverflow.com/questions/8749971/can-i-query-mongodb-objectid-by-date
+    # Convert string date to Date object (otherwise assume timestamp is a date)
+    timestamp = new Date(timestamp) if typeof(timestamp) == 'string'
+    # Convert date object to hex seconds since Unix epoch
+    hexSeconds = Math.floor(timestamp/1000).toString(16)
+    # Create an ObjectId with that hex timestamp
+    mongoose.Types.ObjectId(hexSeconds + "0000000000000000")

server/user_code_problems/user_code_problem_handler.coffee

@@ -1,5 +1,6 @@
 UserCodeProblem = require './UserCodeProblem'
 Handler = require '../commons/Handler'
+utils = require '../lib/utils'
 
 class UserCodeProblemHandler extends Handler
   modelClass: UserCodeProblem
@@ -53,8 +54,8 @@ class UserCodeProblemHandler extends Handler
 
     # Build query
     match = if startDay? or endDay? then {$match: {$and: []}} else {$match: {}}
-    match["$match"]["$and"].push created: {$gte: new Date(startDay + "T00:00:00.000Z")} if startDay?
-    match["$match"]["$and"].push created: {$lt: new Date(endDay + "T00:00:00.000Z")} if endDay?
+    match["$match"]["$and"].push _id: {$gte: utils.objectIdFromTimestamp(startDay + "T00:00:00.000Z")} if startDay?
+    match["$match"]["$and"].push _id: {$lt: utils.objectIdFromTimestamp(endDay + "T00:00:00.000Z")} if endDay?
     group = {"$group": {"_id": {"errMessage": "$errMessageNoLineInfo", "errHint": "$errHint", "language": "$language", "levelID": "$levelID"}, "count": {"$sum": 1}}}
     sort = { $sort : { "_id.levelID": 1, count : -1, "_id.language": 1 } }
     query = UserCodeProblem.aggregate match, group, sort