mirror of
https://github.com/codeninjasllc/codecombat.git
synced 2025-04-26 22:13:32 -04:00
Updating internal analytics scripts
Result of some data sleuthing. They are only used for investigations, and not production aggregation. Hence, a bit ugly and unfinished.
This commit is contained in:
Matt Lott
parent
b70f05fc9e
commit
c2abefb637
3 changed files with 488 additions and 141 deletions
scripts/analytics
184
scripts/analytics/mixpanelGetEvent.py
Normal file
184
scripts/analytics/mixpanelGetEvent.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
# Get mixpanel event data via export API
|
||||
# Useful for debugging Mixpanel data weirdness
|
||||
|
||||
targetLevels = ['dungeons-of-kithgard', 'the-raised-sword', 'endangered-burl']
|
||||
targetLevels = ['dungeons-of-kithgard']
|
||||
eventFunnel = ['Started Level', 'Saw Victory']
|
||||
# eventFunnel = ['Saw Victory']
|
||||
# eventFunnel = ['Started Level']
|
||||
|
||||
import sys
|
||||
from pprint import pprint
|
||||
from datetime import datetime, timedelta
|
||||
from mixpanel import Mixpanel
|
||||
|
||||
try:
|
||||
import json
|
||||
except ImportError:
|
||||
import simplejson as json
|
||||
|
||||
# NOTE: mixpanel dates are by day and inclusive
|
||||
# E.g. '2014-12-08' is any date that day, up to 2014-12-09 12am
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not len(sys.argv) is 3:
|
||||
print "Script format: <script> <api_key> <api_secret>"
|
||||
else:
|
||||
scriptStart = datetime.now()
|
||||
|
||||
api_key = sys.argv[1]
|
||||
api_secret = sys.argv[2]
|
||||
api = Mixpanel(
|
||||
api_key = api_key,
|
||||
api_secret = api_secret
|
||||
)
|
||||
|
||||
startDate = '2015-01-01'
|
||||
endDate = '2015-01-26'
|
||||
|
||||
startEvent = eventFunnel[0]
|
||||
endEvent = eventFunnel[-1]
|
||||
|
||||
print("Requesting data for {0} to {1}".format(startDate, endDate))
|
||||
data = api.request(['export'], {
|
||||
# 'where': '"539c630f30a67c3b05d98d95" == properties["id"]',
|
||||
# 'where': "('539c630f30a67c3b05d98d95' == properties['id'] or '539c630f30a67c3b05d98d95' == properties['distinct_id'])",
|
||||
'event': eventFunnel,
|
||||
'from_date': startDate,
|
||||
'to_date': endDate
|
||||
})
|
||||
|
||||
|
||||
weirdUserIDs = []
|
||||
eventUsers = {}
|
||||
levelEventUserDayMap = {}
|
||||
levelUserEventDayMap = {}
|
||||
lines = data.split('\n')
|
||||
print "Received %d entries" % len(lines)
|
||||
for line in lines:
|
||||
try:
|
||||
if len(line) is 0: continue
|
||||
eventData = json.loads(line)
|
||||
# pprint(eventData)
|
||||
# break
|
||||
eventName = eventData['event']
|
||||
if not eventName in eventFunnel:
|
||||
print 'Unexpected event ' + eventName
|
||||
break
|
||||
if not 'properties' in eventData:
|
||||
print('no properties, skpping')
|
||||
continue
|
||||
properties = eventData['properties']
|
||||
if not 'distinct_id' in properties:
|
||||
print('no distinct_id, skpping')
|
||||
continue
|
||||
user = properties['distinct_id']
|
||||
if not 'time' in properties:
|
||||
print('no time, skpping')
|
||||
continue
|
||||
time = properties['time']
|
||||
pst = datetime.fromtimestamp(int(properties['time']))
|
||||
utc = pst + timedelta(0, 8 * 60 * 60)
|
||||
dateCreated = utc.isoformat()
|
||||
day = dateCreated[0:10]
|
||||
if day < startDate or day > endDate:
|
||||
print "Skipping {0}".format(day)
|
||||
continue
|
||||
|
||||
if 'levelID' in properties:
|
||||
level = properties['levelID']
|
||||
elif 'level' in properties:
|
||||
level = properties['level'].lower().replace(' ', '-')
|
||||
else:
|
||||
print("Unkonwn level for", eventName)
|
||||
print(properties)
|
||||
break
|
||||
|
||||
if not level in targetLevels: continue
|
||||
|
||||
# if user != "539c630f30a67c3b05d98d95": continue
|
||||
pprint(eventData)
|
||||
|
||||
# if user == "54c1fc3a08652d5305442c6b":
|
||||
# pprint(eventData)
|
||||
# break
|
||||
# if '-' in user:
|
||||
# weirdUserIDs.append(user)
|
||||
# # pprint(eventData)
|
||||
# # break
|
||||
# continue
|
||||
|
||||
# print level
|
||||
|
||||
if not level in levelEventUserDayMap: levelEventUserDayMap[level] = {}
|
||||
if not eventName in levelEventUserDayMap[level]: levelEventUserDayMap[level][eventName] = {}
|
||||
if not user in levelEventUserDayMap[level][eventName] or levelEventUserDayMap[level][eventName][user] > day:
|
||||
levelEventUserDayMap[level][eventName][user] = day
|
||||
|
||||
if not user in eventUsers: eventUsers[user] = True
|
||||
|
||||
if not level in levelUserEventDayMap: levelUserEventDayMap[level] = {}
|
||||
if not user in levelUserEventDayMap[level]: levelUserEventDayMap[level][user] = {}
|
||||
if not eventName in levelUserEventDayMap[level][user] or levelUserEventDayMap[level][user][eventName] > day:
|
||||
levelUserEventDayMap[level][user][eventName] = day
|
||||
|
||||
except:
|
||||
print "Unexpected error:", sys.exc_info()[0]
|
||||
print line
|
||||
break
|
||||
|
||||
# pprint(levelEventUserDayMap)
|
||||
|
||||
print("Weird user IDs: {0}".format(len(weirdUserIDs)))
|
||||
|
||||
for level in levelEventUserDayMap:
|
||||
for event in levelEventUserDayMap[level]:
|
||||
print("{0} {1} {2}".format(level, event, len(levelEventUserDayMap[level][event])))
|
||||
print("Users: {0}".format(len(eventUsers)))
|
||||
|
||||
noStartDayUsers = []
|
||||
levelFunnelData = {}
|
||||
for level in levelUserEventDayMap:
|
||||
for user in levelUserEventDayMap[level]:
|
||||
# 6455
|
||||
# for event in levelUserEventDayMap[level][user]:
|
||||
# day = levelUserEventDayMap[level][user][event]
|
||||
# if not level in levelFunnelData: levelFunnelData[level] = {}
|
||||
# if not day in levelFunnelData[level]: levelFunnelData[level][day] = {}
|
||||
# if not event in levelFunnelData[level][day]: levelFunnelData[level][day][event] = 0
|
||||
# levelFunnelData[level][day][event] += 1
|
||||
|
||||
# 5382
|
||||
funnelStartDay = None
|
||||
for event in levelUserEventDayMap[level][user]:
|
||||
day = levelUserEventDayMap[level][user][event]
|
||||
if not level in levelFunnelData: levelFunnelData[level] = {}
|
||||
if not day in levelFunnelData[level]: levelFunnelData[level][day] = {}
|
||||
if not event in levelFunnelData[level][day]: levelFunnelData[level][day][event] = 0
|
||||
if eventFunnel[0] == event:
|
||||
levelFunnelData[level][day][event] += 1
|
||||
funnelStartDay = day
|
||||
break
|
||||
|
||||
if funnelStartDay:
|
||||
for event in levelUserEventDayMap[level][user]:
|
||||
if not event in levelFunnelData[level][funnelStartDay]:
|
||||
levelFunnelData[level][funnelStartDay][event] = 0
|
||||
if eventFunnel[0] != event:
|
||||
levelFunnelData[level][funnelStartDay][event] += 1
|
||||
for i in range(1, len(eventFunnel)):
|
||||
event = eventFunnel[i]
|
||||
if not event in levelFunnelData[level][funnelStartDay]:
|
||||
levelFunnelData[level][funnelStartDay][event] = 0
|
||||
else:
|
||||
noStartDayUsers.append(user)
|
||||
|
||||
pprint(levelFunnelData)
|
||||
print("No start day count: {0}".format(len(noStartDayUsers)))
|
||||
noStartDayUsers.sort()
|
||||
for i in range(len(noStartDayUsers)):
|
||||
if i > 50: break
|
||||
print(noStartDayUsers[i])
|
||||
|
||||
|
||||
print("Script runtime: {0}".format(datetime.now() - scriptStart))
|
||||
|
|
@ -1,10 +1,13 @@
|
|||
# Calculate level completion rates via mixpanel export API
|
||||
|
||||
# TODO: unique users
|
||||
# TODO: align output
|
||||
# TODO: order output
|
||||
# TODO: why are our 'time' fields in PST time?
|
||||
|
||||
targetLevels = ['dungeons-of-kithgard', 'the-raised-sword', 'endangered-burl']
|
||||
eventFunnel = ['Started Level', 'Saw Victory']
|
||||
|
||||
import sys
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from mixpanel import Mixpanel
|
||||
|
||||
try:
|
||||
|
|
@ -19,23 +22,34 @@ if __name__ == '__main__':
|
|||
if not len(sys.argv) is 3:
|
||||
print "Script format: <script> <api_key> <api_secret>"
|
||||
else:
|
||||
scriptStart = datetime.now()
|
||||
|
||||
api_key = sys.argv[1]
|
||||
api_secret = sys.argv[2]
|
||||
api = Mixpanel(
|
||||
api_key = api_key,
|
||||
api_secret = api_secret
|
||||
)
|
||||
|
||||
startDate = '2014-12-31'
|
||||
endDate = '2015-01-05'
|
||||
|
||||
# startDate = '2015-01-11'
|
||||
# endDate = '2015-01-17'
|
||||
startDate = '2015-01-23'
|
||||
endDate = '2015-01-23'
|
||||
# endDate = '2015-01-28'
|
||||
|
||||
startEvent = eventFunnel[0]
|
||||
endEvent = eventFunnel[-1]
|
||||
|
||||
print("Requesting data for {0} to {1}".format(startDate, endDate))
|
||||
data = api.request(['export'], {
|
||||
'event' : ['Started Level', 'Saw Victory'],
|
||||
'event' : eventFunnel,
|
||||
'from_date' : startDate,
|
||||
'to_date' : endDate
|
||||
})
|
||||
|
||||
levelRates = {}
|
||||
|
||||
|
||||
# Map ordering: level, user, event, day
|
||||
userDataMap = {}
|
||||
lines = data.split('\n')
|
||||
print "Received %d entries" % len(lines)
|
||||
for line in lines:
|
||||
|
|
@ -43,40 +57,102 @@ if __name__ == '__main__':
|
|||
if len(line) is 0: continue
|
||||
eventData = json.loads(line)
|
||||
eventName = eventData['event']
|
||||
if not eventName in ['Started Level', 'Saw Victory']:
|
||||
if not eventName in eventFunnel:
|
||||
print 'Unexpected event ' + eventName
|
||||
break
|
||||
if not 'properties' in eventData: continue
|
||||
properties = eventData['properties']
|
||||
if not 'distinct_id' in properties: continue
|
||||
user = properties['distinct_id']
|
||||
if not 'time' in properties: continue
|
||||
time = properties['time']
|
||||
pst = datetime.fromtimestamp(int(properties['time']))
|
||||
utc = pst + timedelta(0, 8 * 60 * 60)
|
||||
dateCreated = utc.isoformat()
|
||||
day = dateCreated[0:10]
|
||||
if day < startDate or day > endDate:
|
||||
print "Skipping {0}".format(day)
|
||||
continue
|
||||
|
||||
if 'levelID' in properties:
|
||||
levelID = properties['levelID']
|
||||
level = properties['levelID']
|
||||
elif 'level' in properties:
|
||||
levelID = properties['level'].lower().replace(' ', '-')
|
||||
level = properties['level'].lower().replace(' ', '-')
|
||||
else:
|
||||
print("Unkonwn levelID for", eventName)
|
||||
print("Unkonwn level for", eventName)
|
||||
print(properties)
|
||||
break
|
||||
if not levelID in levelRates:
|
||||
levelRates[levelID] = {'started': 0, 'finished': 0}
|
||||
if eventName == 'Started Level':
|
||||
levelRates[levelID]['started'] += 1
|
||||
elif eventName == 'Saw Victory':
|
||||
levelRates[levelID]['finished'] += 1
|
||||
else:
|
||||
print("Unknown event name", eventName)
|
||||
print(eventData)
|
||||
break
|
||||
|
||||
if not level in targetLevels:
|
||||
continue
|
||||
|
||||
# print level
|
||||
|
||||
if not level in userDataMap: userDataMap[level] = {}
|
||||
if not user in userDataMap[level]: userDataMap[level][user] = {}
|
||||
if not eventName in userDataMap[level][user] or userDataMap[level][user][eventName] > day:
|
||||
userDataMap[level][user][eventName] = day
|
||||
except:
|
||||
print "Unexpected error:", sys.exc_info()[0]
|
||||
print line
|
||||
break
|
||||
|
||||
# print(levelRates)
|
||||
for levelID in levelRates:
|
||||
started = levelRates[levelID]['started']
|
||||
finished = levelRates[levelID]['finished']
|
||||
# if not levelID == 'endangered-burl':
|
||||
# continue
|
||||
# print(userDataMap)
|
||||
|
||||
levelFunnelData = {}
|
||||
for level in userDataMap:
|
||||
for user in userDataMap[level]:
|
||||
funnelStartDay = None
|
||||
for event in userDataMap[level][user]:
|
||||
day = userDataMap[level][user][event]
|
||||
if not level in levelFunnelData: levelFunnelData[level] = {}
|
||||
if not day in levelFunnelData[level]: levelFunnelData[level][day] = {}
|
||||
if not event in levelFunnelData[level][day]: levelFunnelData[level][day][event] = 0
|
||||
if eventFunnel[0] == event:
|
||||
levelFunnelData[level][day][event] += 1
|
||||
funnelStartDay = day
|
||||
break
|
||||
|
||||
if funnelStartDay:
|
||||
for event in userDataMap[level][user]:
|
||||
if not event in levelFunnelData[level][funnelStartDay]:
|
||||
levelFunnelData[level][funnelStartDay][event] = 0
|
||||
if not eventFunnel[0] == event:
|
||||
levelFunnelData[level][funnelStartDay][event] += 1
|
||||
for i in range(1, len(eventFunnel)):
|
||||
event = eventFunnel[i]
|
||||
if not event in levelFunnelData[level][funnelStartDay]:
|
||||
levelFunnelData[level][funnelStartDay][event] = 0
|
||||
|
||||
# print(levelFunnelData)
|
||||
|
||||
totals = {}
|
||||
for level in levelFunnelData:
|
||||
for day in levelFunnelData[level]:
|
||||
if startEvent in levelFunnelData[level][day]:
|
||||
started = levelFunnelData[level][day][startEvent]
|
||||
else:
|
||||
started = 0
|
||||
if endEvent in levelFunnelData[level][day]:
|
||||
finished = levelFunnelData[level][day][endEvent]
|
||||
else:
|
||||
finished = 0
|
||||
if not level in totals: totals[level] = {}
|
||||
if not startEvent in totals[level]: totals[level][startEvent] = 0
|
||||
if not endEvent in totals[level]: totals[level][endEvent] = 0
|
||||
totals[level][startEvent] += started
|
||||
totals[level][endEvent] += finished
|
||||
if started > 0:
|
||||
print("{0}\t{1}\t{2}\t{3}\t{4}%".format(level, day, started, finished, float(finished) / started * 100))
|
||||
else:
|
||||
print("{0}\t{1}\t{2}\t{3}\t".format(level, day, started, finished))
|
||||
|
||||
for level in totals:
|
||||
started = totals[level][startEvent]
|
||||
finished = totals[level][endEvent]
|
||||
if started > 0:
|
||||
print("{0}\t{1}\t{2}\t{3}%".format(levelID, started, finished, float(finished) / started * 100))
|
||||
print("{0}\t{1}\t{2}\t{3}%".format(level, started, finished, float(finished) / started * 100))
|
||||
else:
|
||||
print("{0}\t{1}\t{2}".format(levelID, started, finished))
|
||||
print("{0}\t{1}\t{2}\t".format(level, started, finished))
|
||||
|
||||
print("Script runtime: {0}".format(datetime.now() - scriptStart))
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@
|
|||
|
||||
// TODO: Why do a small number of 'Started level' not have properties.levelID set?
|
||||
|
||||
// TODO: spot check the data: NaN, only some 0.0 dates, etc.
|
||||
// TODO: exclude levels with no interesting data?
|
||||
// TODO: Fix addPlaytimeAverages() and addUserCodeProblemCounts()
|
||||
// TODO: getLevelFunnelData() outputs different data structure now.
|
||||
|
||||
var startTime = new Date();
|
||||
|
||||
|
|
@ -20,12 +20,22 @@ var today = new Date();
|
|||
today = today.toISOString().substr(0, 10);
|
||||
print("Today is " + today);
|
||||
|
||||
var todayMinus6 = new Date();
|
||||
todayMinus6.setUTCDate(todayMinus6.getUTCDate() - 6);
|
||||
var startDate = todayMinus6.toISOString().substr(0, 10) + "T00:00:00.000Z";
|
||||
print("Start date is " + startDate)
|
||||
// startDate = "2015-01-02T00:00:00.000Z";
|
||||
// var endDate = "2015-01-09T00:00:00.000Z";
|
||||
// var todayMinus6 = new Date();
|
||||
// todayMinus6.setUTCDate(todayMinus6.getUTCDate() - 6);
|
||||
// var startDate = todayMinus6.toISOString().substr(0, 10) + "T00:00:00.000Z";
|
||||
// startDate = "2015-01-23T00:00:00.000Z";
|
||||
// print("Start date is " + startDate)
|
||||
// var endDate = "2015-01-24T00:00:00.000Z";
|
||||
// print("End date is " + endDate)
|
||||
|
||||
var levelCompletionFunnel = ['Started Level', 'Saw Victory'];
|
||||
var dataStartDay = "2015-01-15";
|
||||
var startDay = "2015-01-23";
|
||||
var endDay = "2015-01-24";
|
||||
print(startDay + " to " + endDay);
|
||||
print("Data start day " + dataStartDay);
|
||||
|
||||
var targetLevels = ['dungeons-of-kithgard'];
|
||||
|
||||
function objectIdWithTimestamp(timestamp)
|
||||
{
|
||||
|
|
@ -38,72 +48,139 @@ function objectIdWithTimestamp(timestamp)
|
|||
return constructedObjectId
|
||||
}
|
||||
|
||||
function getCompletionRates() {
|
||||
print("Getting completion rates...");
|
||||
var queryParams = {
|
||||
$and: [
|
||||
{_id: {$gte: objectIdWithTimestamp(ISODate(startDate))}},
|
||||
{$or: [ {"event" : 'Started Level'}, {"event" : 'Saw Victory'}]}
|
||||
]
|
||||
};
|
||||
function getLevelFunnelData(startDay, endDay, eventFunnel) {
|
||||
// Copied from insertPerDayAnalytics.js
|
||||
if (!startDay || !eventFunnel || eventFunnel.length === 0) return {};
|
||||
|
||||
// var startObj = objectIdWithTimestamp(ISODate(startDay + "T00:00:00.000Z"));
|
||||
var startObj = objectIdWithTimestamp(ISODate(dataStartDay + "T00:00:00.000Z"));
|
||||
var endObj = objectIdWithTimestamp(ISODate(endDay + "T00:00:00.000Z"));
|
||||
var queryParams = {$and: [{_id: {$gte: startObj}},{_id: {$lt: endObj}},{"event": {$in: eventFunnel}}]};
|
||||
// var queryParams = {$and: [{user: ObjectId("539c630f30a67c3b05d98d95")},{_id: {$gte: startObj}},{_id: {$lt: endObj}},{"event": {$in: eventFunnel}}]};
|
||||
var cursor = db['analytics.log.events'].find(queryParams);
|
||||
|
||||
// <level><date><data>
|
||||
var levelData = {};
|
||||
// Map ordering: level, user, event, day
|
||||
var recordCount = 0;
|
||||
var duplicates = {};
|
||||
var levelEventUserDayMap = {};
|
||||
var levelUserEventDayMap = {};
|
||||
while (cursor.hasNext()) {
|
||||
recordCount++;
|
||||
var doc = cursor.next();
|
||||
var created = doc.created.toISOString().substring(0, 10);
|
||||
var created = doc._id.getTimestamp().toISOString();
|
||||
var day = created.substring(0, 10);
|
||||
var event = doc.event;
|
||||
var properties = doc.properties;
|
||||
var user = doc.user;
|
||||
var level;
|
||||
|
||||
// TODO: Switch to properties.levelID for 'Saw Victory'
|
||||
if (event === 'Saw Victory' && properties.level) level = properties.level.toLowerCase().replace(/ /g, '-');
|
||||
else if (properties.levelID) level = properties.levelID
|
||||
else continue
|
||||
var user = doc.user;
|
||||
|
||||
// if (targetLevels.indexOf(level) < 0) continue;
|
||||
|
||||
// print(day + " " + created);
|
||||
// print(JSON.stringify(doc, null, 2));
|
||||
|
||||
if (level.length > longestLevelName) longestLevelName = level.length;
|
||||
|
||||
if (!levelData[level]) levelData[level] = {};
|
||||
if (!levelData[level][created]) levelData[level][created] = {};
|
||||
if (!levelData[level][created]['started']) levelData[level][created]['started'] = {};
|
||||
if (!levelData[level][created]['finished']) levelData[level][created]['finished'] = {}
|
||||
if (event === 'Started Level') levelData[level][created]['started'][user] = true;
|
||||
else levelData[level][created]['finished'][user] = true;
|
||||
if (!levelUserEventDayMap[level]) levelUserEventDayMap[level] = {};
|
||||
if (!levelUserEventDayMap[level][user]) levelUserEventDayMap[level][user] = {};
|
||||
if (levelUserEventDayMap[level][user][event]) {
|
||||
if (!duplicates[event]) duplicates[event] = 0;
|
||||
duplicates[event]++;
|
||||
}
|
||||
if (!levelUserEventDayMap[level][user][event] || levelUserEventDayMap[level][user][event].localeCompare(day) > 0) {
|
||||
// if (!levelUserEventDayMap[level][user][event] || day.localeCompare(levelUserEventDayMap[level][user][event]) > 0) {
|
||||
// day is earlier than levelUserEventDayMap[level][user][event]
|
||||
levelUserEventDayMap[level][user][event] = day;
|
||||
}
|
||||
|
||||
if (!levelEventUserDayMap[level]) levelEventUserDayMap[level] = {};
|
||||
if (!levelEventUserDayMap[level][event]) levelEventUserDayMap[level][event] = {};
|
||||
if (!levelEventUserDayMap[level][event][user] || levelEventUserDayMap[level][event][user].localeCompare(day) > 0) {
|
||||
levelEventUserDayMap[level][event][user] = day;
|
||||
}
|
||||
}
|
||||
|
||||
// print("Records: " + recordCount);
|
||||
// print("Duplicates");
|
||||
// print(JSON.stringify(duplicates, null, 2));
|
||||
longestLevelName += 2;
|
||||
|
||||
var levelRates = [];
|
||||
for (level in levelData) {
|
||||
var dateData = [];
|
||||
var dateIndex = 0;
|
||||
for (created in levelData[level]) {
|
||||
var started =
|
||||
dateData.push({
|
||||
level: level,
|
||||
created: created,
|
||||
started: Object.keys(levelData[level][created]['started']).length,
|
||||
finished: Object.keys(levelData[level][created]['finished']).length
|
||||
});
|
||||
if (dates.length === dateIndex) dates.push(created.substring(5));
|
||||
dateIndex++;
|
||||
// Data: level, day, event
|
||||
var noStartDayUsers = [];
|
||||
var levelFunnelData = {};
|
||||
for (level in levelUserEventDayMap) {
|
||||
for (user in levelUserEventDayMap[level]) {
|
||||
|
||||
// Find first event date
|
||||
var funnelStartDay = null;
|
||||
for (event in levelUserEventDayMap[level][user]) {
|
||||
var day = levelUserEventDayMap[level][user][event];
|
||||
if (day.localeCompare(startDay) < 0) {
|
||||
// day earlier than startDay
|
||||
continue;
|
||||
}
|
||||
if (!levelFunnelData[level]) levelFunnelData[level] = {};
|
||||
if (!levelFunnelData[level][day]) levelFunnelData[level][day] = {};
|
||||
if (!levelFunnelData[level][day][event]) levelFunnelData[level][day][event] = 0;
|
||||
if (eventFunnel[0] === event) {
|
||||
// First event gets attributed to current date
|
||||
levelFunnelData[level][day][event]++;
|
||||
funnelStartDay = day;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (funnelStartDay) {
|
||||
// Add remaining funnel steps/events to first step's date
|
||||
for (event in levelUserEventDayMap[level][user]) {
|
||||
if (!levelFunnelData[level][funnelStartDay][event]) levelFunnelData[level][funnelStartDay][event] = 0;
|
||||
if (eventFunnel[0] != event) levelFunnelData[level][funnelStartDay][event]++;
|
||||
}
|
||||
// Zero remaining funnel events
|
||||
for (var i = 1; i < eventFunnel.length; i++) {
|
||||
var event = eventFunnel[i];
|
||||
if (!levelFunnelData[level][funnelStartDay][event]) levelFunnelData[level][funnelStartDay][event] = 0;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// TODO: calc no start days
|
||||
for (event in levelUserEventDayMap[level][user]) {
|
||||
var day = levelUserEventDayMap[level][user][event];
|
||||
if (day.localeCompare(startDay) < 0) {
|
||||
// day earlier than startDay
|
||||
continue;
|
||||
}
|
||||
if (eventFunnel[0] != event) {
|
||||
noStartDayUsers.push(user);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
levelRates.push(dateData);
|
||||
}
|
||||
// printjson(levelRates);
|
||||
|
||||
levelRates.sort(function(a,b) {return a[0].level < b[0].level ? -1 : 1});
|
||||
for (levelKey in levelRates) levelRates[levelKey].sort(function(a,b) {return a.created < b.created ? 1 : -1});
|
||||
// print("No start day count: " + noStartDayUsers.length);
|
||||
// for (var i = 0; i < noStartDayUsers.length && i < 50; i++) {
|
||||
// print(noStartDayUsers[i]);
|
||||
// }
|
||||
|
||||
return levelRates;
|
||||
return levelFunnelData;
|
||||
}
|
||||
|
||||
function addPlaytimeAverages(levelRates) {
|
||||
function addPlaytimeAverages(startDay, endDay, levelRates) {
|
||||
print("Getting playtimes...");
|
||||
// printjson(levelRates);
|
||||
var startObj = objectIdWithTimestamp(ISODate(dataStartDay + "T00:00:00.000Z"));
|
||||
var endObj = objectIdWithTimestamp(ISODate(endDay + "T00:00:00.000Z"));
|
||||
// var match = {"$match" : {$and: [{_id: { $gte: startObj}}, {_id: { $lt: endObj}}]}};
|
||||
var match = {
|
||||
"$match" : {
|
||||
$and: [
|
||||
{"created": { $gte: ISODate(startDate)}},
|
||||
{_id: { $gte: startObj}},
|
||||
{_id: { $lt: endObj}},
|
||||
{"state.complete": true},
|
||||
{"playtime": {$gt: 0}}
|
||||
]
|
||||
|
|
@ -113,12 +190,12 @@ function addPlaytimeAverages(levelRates) {
|
|||
"_id" : 0,
|
||||
"levelID" : 1,
|
||||
"playtime": 1,
|
||||
"created": {"$substr" : ["$created", 0, 10]}
|
||||
"day": {"$substr" : ["$created", 0, 10]}
|
||||
}};
|
||||
|
||||
var group = {"$group" : {
|
||||
"_id" : {
|
||||
"created" : "$created",
|
||||
"day" : "$day",
|
||||
"level": "$levelID"
|
||||
},
|
||||
"average" : {
|
||||
|
|
@ -131,36 +208,38 @@ function addPlaytimeAverages(levelRates) {
|
|||
var levelPlaytimeData = {};
|
||||
while (cursor.hasNext()) {
|
||||
var doc = cursor.next();
|
||||
var created = doc._id.created;
|
||||
var day = doc._id.day;
|
||||
var level = doc._id.level;
|
||||
if (!levelPlaytimeData[level]) levelPlaytimeData[level] = {};
|
||||
levelPlaytimeData[level][created] = doc.average;
|
||||
levelPlaytimeData[level][day] = doc.average;
|
||||
}
|
||||
|
||||
for (levelIndex in levelRates) {
|
||||
for (dateIndex in levelRates[levelIndex]) {
|
||||
var level = levelRates[levelIndex][dateIndex].level;
|
||||
var created = levelRates[levelIndex][dateIndex].created;
|
||||
if (levelPlaytimeData[level] && levelPlaytimeData[level][created]) {
|
||||
levelRates[levelIndex][dateIndex].averagePlaytime = levelPlaytimeData[level][created];
|
||||
var day = levelRates[levelIndex][dateIndex].day;
|
||||
if (levelPlaytimeData[level] && levelPlaytimeData[level][day]) {
|
||||
levelRates[levelIndex][dateIndex].averagePlaytime = levelPlaytimeData[level][day];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function addUserCodeProblemCounts(levelRates) {
|
||||
function addUserCodeProblemCounts(startDay, endDay, levelRates) {
|
||||
print("Getting user code problem counts...");
|
||||
var match = {"$match" : {"created": { $gte: ISODate(startDate)}}};
|
||||
var startObj = objectIdWithTimestamp(ISODate(dataStartDay + "T00:00:00.000Z"));
|
||||
var endObj = objectIdWithTimestamp(ISODate(endDay + "T00:00:00.000Z"));
|
||||
var match = {"$match" : {$and: [{_id: { $gte: startObj}}, {_id: { $lt: endObj}}]}};
|
||||
|
||||
var proj0 = {"$project": {
|
||||
"_id" : 0,
|
||||
"levelID" : 1,
|
||||
"created": {"$substr" : ["$created", 0, 10]}
|
||||
"day": {"$substr" : ["$created", 0, 10]}
|
||||
}};
|
||||
|
||||
var group = {"$group" : {
|
||||
"_id" : {
|
||||
"created" : "$created",
|
||||
"day" : "$day",
|
||||
"level": "$levelID"
|
||||
},
|
||||
"count" : {
|
||||
|
|
@ -173,18 +252,18 @@ function addUserCodeProblemCounts(levelRates) {
|
|||
var levelPlaytimeData = {};
|
||||
while (cursor.hasNext()) {
|
||||
var doc = cursor.next();
|
||||
var created = doc._id.created;
|
||||
var day = doc._id.day;
|
||||
var level = doc._id.level;
|
||||
if (!levelPlaytimeData[level]) levelPlaytimeData[level] = {};
|
||||
levelPlaytimeData[level][created] = doc.count;
|
||||
levelPlaytimeData[level][day] = doc.count;
|
||||
}
|
||||
|
||||
for (levelIndex in levelRates) {
|
||||
for (dateIndex in levelRates[levelIndex]) {
|
||||
var level = levelRates[levelIndex][dateIndex].level;
|
||||
var created = levelRates[levelIndex][dateIndex].created;
|
||||
if (levelPlaytimeData[level] && levelPlaytimeData[level][created]) {
|
||||
levelRates[levelIndex][dateIndex].codeProblems = levelPlaytimeData[level][created];
|
||||
var day = levelRates[levelIndex][dateIndex].day;
|
||||
if (levelPlaytimeData[level] && levelPlaytimeData[level][day]) {
|
||||
levelRates[levelIndex][dateIndex].codeProblems = levelPlaytimeData[level][day];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -193,60 +272,68 @@ function addUserCodeProblemCounts(levelRates) {
|
|||
var longestLevelName = -1;
|
||||
var dates = [];
|
||||
|
||||
var levelRates = getCompletionRates();
|
||||
// addPlaytimeAverages(levelRates);
|
||||
// addUserCodeProblemCounts(levelRates);
|
||||
var levelRates = getLevelFunnelData(startDay, endDay, levelCompletionFunnel);
|
||||
// addPlaytimeAverages(startDay, endDay, levelRates);
|
||||
// addUserCodeProblemCounts(startDay, endDay, levelRates);
|
||||
|
||||
// print(JSON.stringify(levelRates, null, 2))
|
||||
|
||||
// Print out all data
|
||||
print("Columns: level, day, started, finished, completion rate, average finish playtime, average code problem count");
|
||||
// print("Columns: level, day, started, finished, completion rate, average finish playtime, average code problem count");
|
||||
print("Columns: level, day, started, finished, completion rate");
|
||||
for (levelKey in levelRates) {
|
||||
for (dateKey in levelRates[levelKey]) {
|
||||
var created = levelRates[levelKey][dateKey].created;
|
||||
var level = levelRates[levelKey][dateKey].level;
|
||||
var started = levelRates[levelKey][dateKey].started;
|
||||
var finished = levelRates[levelKey][dateKey].finished;
|
||||
// var day = levelRates[levelKey][dateKey].day;
|
||||
// var level = levelRates[levelKey][dateKey].level;
|
||||
// var started = levelRates[levelKey][dateKey].started;
|
||||
// var finished = levelRates[levelKey][dateKey].finished;
|
||||
var started = levelRates[levelKey][dateKey][levelCompletionFunnel[0]] || 0;
|
||||
var finished = levelRates[levelKey][dateKey][levelCompletionFunnel[levelCompletionFunnel.length - 1]] || 0;
|
||||
var completionRate = started > 0 ? finished / started : 0;
|
||||
var averagePlaytime = levelRates[levelKey][dateKey].averagePlaytime;
|
||||
averagePlaytime = averagePlaytime ? Math.round(averagePlaytime) : 0;
|
||||
var averageCodeProblems = levelRates[levelKey][dateKey].codeProblems;
|
||||
averageCodeProblems = averageCodeProblems ? (averageCodeProblems / started).toFixed(2) : 0.0;
|
||||
var levelSpacer = new Array(longestLevelName - level.length).join(' ');
|
||||
print(level + levelSpacer + created + "\t" + started + "\t" + finished + "\t" + (completionRate * 100).toFixed(2) + "% " + averagePlaytime + "s " + averageCodeProblems);
|
||||
// var averagePlaytime = levelRates[levelKey][dateKey].averagePlaytime;
|
||||
// averagePlaytime = averagePlaytime ? Math.round(averagePlaytime) : 0;
|
||||
// var averageCodeProblems = levelRates[levelKey][dateKey].codeProblems;
|
||||
// averageCodeProblems = averageCodeProblems ? (averageCodeProblems / started).toFixed(2) : 0.0;
|
||||
if ((longestLevelName - levelKey.length) < 0)
|
||||
throw new Error(longestLevelName + " " + levelKey.length);
|
||||
var levelSpacer = new Array(longestLevelName - levelKey.length).join(' ');
|
||||
// print(levelKey + levelSpacer + dateKey + "\t" + started + "\t" + finished + "\t" + (completionRate * 100).toFixed(2) + "% " + averagePlaytime + "s " + averageCodeProblems);
|
||||
print(levelKey + levelSpacer + dateKey + "\t" + started + "\t" + finished + "\t" + (completionRate * 100).toFixed(2) + "%");
|
||||
}
|
||||
}
|
||||
|
||||
// Print out a nice grid of levels with 7 days of data
|
||||
print("Columns: level, completion rate/average playtime/average code problems, completion rate/average playtime/average code problems ...");
|
||||
print(new Array(longestLevelName).join(' ') + dates.join('\t\t'));
|
||||
for (levelKey in levelRates) {
|
||||
var hasStarted = false;
|
||||
for (dateKey in levelRates[levelKey]) {
|
||||
if (levelRates[levelKey][dateKey].started > 0) {
|
||||
hasStarted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!hasStarted) continue;
|
||||
|
||||
if (levelRates[levelKey].length < 6) continue;
|
||||
|
||||
var level = levelRates[levelKey][0].level;
|
||||
var levelSpacer = new Array(longestLevelName - level.length).join(' ');
|
||||
var msg = level + levelSpacer;
|
||||
|
||||
for (dateKey in levelRates[levelKey]) {
|
||||
var created = levelRates[levelKey][dateKey].created;
|
||||
var started = levelRates[levelKey][dateKey].started;
|
||||
var finished = levelRates[levelKey][dateKey].finished;
|
||||
var averagePlaytime = levelRates[levelKey][dateKey].averagePlaytime;
|
||||
averagePlaytime = averagePlaytime ? Math.round(averagePlaytime) : 0;
|
||||
var averageCodeProblems = levelRates[levelKey][dateKey].codeProblems;
|
||||
averageCodeProblems = averageCodeProblems ? averageCodeProblems / started : 0.0;
|
||||
var completionRate = started > 0 ? finished / started : 0;
|
||||
msg += (completionRate * 100).toFixed(2) + "/" + averagePlaytime + "/" + averageCodeProblems.toFixed(2) + "\t";
|
||||
}
|
||||
print(msg);
|
||||
}
|
||||
// print("Columns: level, completion rate/average playtime/average code problems, completion rate/average playtime/average code problems ...");
|
||||
// print(new Array(longestLevelName).join(' ') + dates.join('\t\t'));
|
||||
// for (levelKey in levelRates) {
|
||||
// var hasStarted = false;
|
||||
// for (dateKey in levelRates[levelKey]) {
|
||||
// if (levelRates[levelKey][dateKey].started > 0) {
|
||||
// hasStarted = true;
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// if (!hasStarted) continue;
|
||||
//
|
||||
// if (levelRates[levelKey].length < 6) continue;
|
||||
//
|
||||
// var level = levelRates[levelKey][0].level;
|
||||
// var levelSpacer = new Array(longestLevelName - level.length).join(' ');
|
||||
// var msg = level + levelSpacer;
|
||||
//
|
||||
// for (dateKey in levelRates[levelKey]) {
|
||||
// var day = levelRates[levelKey][dateKey].day;
|
||||
// var started = levelRates[levelKey][dateKey].started;
|
||||
// var finished = levelRates[levelKey][dateKey].finished;
|
||||
// var averagePlaytime = levelRates[levelKey][dateKey].averagePlaytime;
|
||||
// averagePlaytime = averagePlaytime ? Math.round(averagePlaytime) : 0;
|
||||
// var averageCodeProblems = levelRates[levelKey][dateKey].codeProblems;
|
||||
// averageCodeProblems = averageCodeProblems ? averageCodeProblems / started : 0.0;
|
||||
// var completionRate = started > 0 ? finished / started : 0;
|
||||
// msg += (completionRate * 100).toFixed(2) + "/" + averagePlaytime + "/" + averageCodeProblems.toFixed(2) + "\t";
|
||||
// }
|
||||
// print(msg);
|
||||
// }
|
||||
|
||||
var endTime = new Date();
|
||||
print("Runtime: " + (endTime - startTime));
|
||||
print("Runtime: " + (endTime - startTime));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue