codeninjasllc/codecombat
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add browser segments to a/b test scripts

Browse source
This commit is contained in:
Matt Lott 2015-02-11 11:07:32 -08:00
parent 9ead6600b0
commit 6178800d16
5 changed files with 212 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

47
scripts/analytics/mongodb/queries/abForeshadowsLevels.js
View file

@ -1,21 +1,15 @@
// foreshadowsLevels A/B Results
// Test started 2015-01-29
// Fixed some particle problems on 2015-02-02
// Usage:
// mongo <address>:<port>/<database> <script file> -u <username> -p <password>
// Inputs to modify below:
// numDays - number of days into the past to fetch
// eventFunnel - ordered array of events that define the completion funnel
// levelSlugs - [optional] array of levels to examine, otherwise fetch all levels
// testGroupFn - return group value from user testGroupNumber
// Include getFunnelData(), log()
load('abTestHelpers.js');
var scriptStartTime = new Date();
try {
var startDay = '2015-01-29'
var startDay = '2015-02-03'
log("Today is " + new Date().toISOString().substr(0, 10));
log("Start day is " + startDay);
@ -30,33 +24,20 @@ try {
var funnelData = getFunnelData(startDay, eventFunnel, testGroupFn, levelSlugs);
log("Day\tLevel\tGroup\tStarted\tFinished\tCompletion Rate");
var overallCounts = {};
for (var i = 0; i < funnelData.length; i++) {
var level = funnelData[i].level;
var day = funnelData[i].day;
var group = funnelData[i].group;
var started = funnelData[i].started;
var finished = funnelData[i].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
if (!overallCounts[level]) overallCounts[level] = {};
if (!overallCounts[level][group]) overallCounts[level][group] = {started: 0, finished: 0};
overallCounts[level][group]['started'] += started;
overallCounts[level][group]['finished'] += finished;
log(day + "\t" + level + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
log("Overall totals:");
for (level in overallCounts) {
for (group in overallCounts[level]) {
var started = overallCounts[level][group].started;
var finished = overallCounts[level][group].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
printFunnelData(funnelData, function (day, level, browser, group, started, finished, rate) {
if (day && level && browser && group) {
log(day + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
else if (level && browser && group) {
log(level + "\t" + browser + "\t" + (browser.length < 8 ? "\t": "") + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
else if (level && group) {
log(level + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
}
else if (group) {
log(group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
});
}
catch(err) {
log("ERROR: " + err);

48
scripts/analytics/mongodb/queries/abGemPromptGroup.js
View file

@ -4,13 +4,6 @@
// Usage:
// mongo <address>:<port>/<database> <script file> -u <username> -p <password>
// Inputs to modify below:
// numDays - number of days into the past to fetch
// eventFunnel - ordered array of events that define the completion funnel
// levelSlugs - [optional] array of levels to examine, otherwise fetch all levels
// testGroupFn - return group value from user testGroupNumber
// Include getFunnelData(), log()
load('abTestHelpers.js');
var scriptStartTime = new Date();
@ -29,43 +22,14 @@ try {
var funnelData = getFunnelData(startDay, eventFunnel, testGroupFn);
log("Day\t\tGroup\t\tStarted\tFinished\tCompletion Rate");
var overallCounts = {};
for (var i = 0; i < funnelData.length; i++) {
var level = funnelData[i].level;
var day = funnelData[i].day;
var group = funnelData[i].group;
var started = funnelData[i].started;
var finished = funnelData[i].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
if (!overallCounts[level]) overallCounts[level] = {};
if (!overallCounts[level][group]) overallCounts[level][group] = {started: 0, finished: 0};
overallCounts[level][group]['started'] += started;
overallCounts[level][group]['finished'] += finished;
if (group === 'prompt') {
log(day + "\t" + group + "\t\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
printFunnelData(funnelData, function (day, level, browser, group, started, finished, rate) {
if (day && level && browser && group) {
log(day + "\t" + group + "\t" + (group === 'prompt' ? "\t": "") + started + "\t" + finished + "\t" + rate.toFixed(2));
}
else {
log(day + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
else if (group) {
log(group + (group === 'prompt' ? "\t": "") + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
}
log("Overall totals:");
for (level in overallCounts) {
for (group in overallCounts[level]) {
var started = overallCounts[level][group].started;
var finished = overallCounts[level][group].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
if (group === 'prompt') {
log(group + "\t\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
else {
log(group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
}
}
});
}
catch(err) {
log("ERROR: " + err);

44
scripts/analytics/mongodb/queries/abLeaderboardsGroup.js
View file

@ -4,13 +4,6 @@
// Usage:
// mongo <address>:<port>/<database> <script file> -u <username> -p <password>
// Inputs to modify below:
// numDays - number of days into the past to fetch
// eventFunnel - ordered array of events that define the completion funnel
// levelSlugs - [optional] array of levels to examine, otherwise fetch all levels
// testGroupFn - return group value from user testGroupNumber
// Include getFunnelData(), log()
load('abTestHelpers.js');
var scriptStartTime = new Date();
@ -33,33 +26,20 @@ try {
var funnelData = getFunnelData(startDay, eventFunnel, testGroupFn, levelSlugs);
log("Day\tLevel\tGroup\tStarted\tFinished\tCompletion Rate");
var overallCounts = {};
for (var i = 0; i < funnelData.length; i++) {
var level = funnelData[i].level;
var day = funnelData[i].day;
var group = funnelData[i].group;
var started = funnelData[i].started;
var finished = funnelData[i].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
if (!overallCounts[level]) overallCounts[level] = {};
if (!overallCounts[level][group]) overallCounts[level][group] = {started: 0, finished: 0};
overallCounts[level][group]['started'] += started;
overallCounts[level][group]['finished'] += finished;
log(day + "\t" + level + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
log("Overall totals:");
for (level in overallCounts) {
for (group in overallCounts[level]) {
var started = overallCounts[level][group].started;
var finished = overallCounts[level][group].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
printFunnelData(funnelData, function (day, level, browser, group, started, finished, rate) {
if (day && level && browser && group) {
log(day + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
else if (level && browser && group) {
log(level + "\t" + browser + "\t" + (browser.length < 8 ? "\t": "") + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
else if (level && group) {
log(level + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
}
else if (group) {
log(group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
});
}
catch(err) {
log("ERROR: " + err);

44
scripts/analytics/mongodb/queries/abShowsPortal.js
View file

@ -4,13 +4,6 @@
// Usage:
// mongo <address>:<port>/<database> <script file> -u <username> -p <password>
// Inputs to modify below:
// numDays - number of days into the past to fetch
// eventFunnel - ordered array of events that define the completion funnel
// levelSlugs - [optional] array of levels to examine, otherwise fetch all levels
// testGroupFn - return group value from user testGroupNumber
// Include getFunnelData(), log()
load('abTestHelpers.js');
var scriptStartTime = new Date();
@ -29,33 +22,20 @@ try {
var funnelData = getFunnelData(startDay, eventFunnel, testGroupFn, levelSlugs);
log("Day\tLevel\tGroup\tStarted\tFinished\tCompletion Rate");
var overallCounts = {};
for (var i = 0; i < funnelData.length; i++) {
var level = funnelData[i].level;
var day = funnelData[i].day;
var group = funnelData[i].group;
var started = funnelData[i].started;
var finished = funnelData[i].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
if (!overallCounts[level]) overallCounts[level] = {};
if (!overallCounts[level][group]) overallCounts[level][group] = {started: 0, finished: 0};
overallCounts[level][group]['started'] += started;
overallCounts[level][group]['finished'] += finished;
log(day + "\t" + level + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
log("Overall totals:");
for (level in overallCounts) {
for (group in overallCounts[level]) {
var started = overallCounts[level][group].started;
var finished = overallCounts[level][group].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
printFunnelData(funnelData, function (day, level, browser, group, started, finished, rate) {
if (day && level && browser && group) {
log(day + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
else if (level && browser && group) {
log(level + "\t" + browser + "\t" + (browser.length < 8 ? "\t": "") + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
else if (level && group) {
log(level + "\t" + group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
}
else if (group) {
log(group + "\t" + started + "\t" + finished + "\t" + rate.toFixed(2));
}
});
}
catch(err) {
log("ERROR: " + err);

207
scripts/analytics/mongodb/queries/abTestHelpers.js
View file

@ -1,11 +1,13 @@
// A/B test helper functions
// Loaded from ab*.js ab test result scripts
// Main API is getFunnelData() which returns per-day funnel completion rates
// Main APIs are getFunnelData() and printFunnelData()
// TODO: use levelSlugs in query if available
// TODO: Stop looking up testGroupNumber when test group data is available in analytics.log.events
// TODO: These are super slow, need to aggregate into analytics.perdays collection
var browserCountPrintThreshold = 1000;
var analyticsStringCache = {};
var analyticsStringIDCache = {};
@ -56,8 +58,10 @@ function getFunnelData(startDay, eventFunnel, testGroupFn, levelSlugs) {
var queryParams = {$and: [{_id: {$gte: startObj}},{"event": {$in: eventFunnel}}]};
var cursor = db['analytics.log.events'].find(queryParams);
log("Fetching events..");
// Map ordering: level, user, event, day
var levelUserEventMap = {};
var levelSessions = [];
var users = [];
while (cursor.hasNext()) {
var doc = cursor.next();
@ -66,27 +70,36 @@ function getFunnelData(startDay, eventFunnel, testGroupFn, levelSlugs) {
var event = doc.event;
var properties = doc.properties;
var user = doc.user.valueOf();
var level;
var level = 'n/a';
var ls = null;
// TODO: Switch to properties.levelID for 'Saw Victory'
if (event === 'Saw Victory' && properties.level) level = properties.level.toLowerCase().replace(/ /g, '-');
else if (properties.levelID) level = properties.levelID
else level = 'n/a'
if (levelSlugs && levelSlugs.indexOf(level) < 0) continue;
if (properties && properties.ls) {
ls = properties.ls.valueOf();
levelSessions.push(properties.ls);
}
users.push(ObjectId(user));
if (!levelUserEventMap[level]) levelUserEventMap[level] = {};
if (!levelUserEventMap[level][user]) levelUserEventMap[level][user] = {};
if (!levelUserEventMap[level][user][event]
|| levelUserEventMap[level][user][event].localeCompare(day) > 0) {
levelUserEventMap[level][user][event] = day;
|| levelUserEventMap[level][user][event]['day'].localeCompare(day) > 0) {
levelUserEventMap[level][user][event] = {day: day};
if (ls) {
levelUserEventMap[level][user][event]['ls'] = ls;
}
}
}
// printjson(levelUserEventMap);
// printjson(users);
log("Fetching users..");
var userGroupMap = {};
cursor = db['users'].find({_id : {$in: users}});
while (cursor.hasNext()) {
@ -96,77 +109,122 @@ function getFunnelData(startDay, eventFunnel, testGroupFn, levelSlugs) {
}
// printjson(userGroupMap);
log("Fetching level sessions..");
var lsBrowserMap = {};
var userBrowserMap = {};
cursor = db['level.sessions'].find({_id : {$in: levelSessions}});
while (cursor.hasNext()) {
var doc = cursor.next();
var user = doc._id.valueOf();
var browser = doc.browser;
var browserInfo = '';
if (browser && browser.platform) {
browserInfo += browser.platform;
}
if (browser && browser.name) {
browserInfo += browser.name;
}
if (browserInfo.length > 0) {
lsBrowserMap[doc._id.valueOf()] = browserInfo;
userBrowserMap[user] = browserInfo;
}
}
// printjson(lsBrowserMap);
log("Mapping data..");
// Data: level, day, event
var levelDayGroupEventMap = {};
var levelDayGroupBrowserEventMap = {};
for (level in levelUserEventMap) {
for (user in levelUserEventMap[level]) {
var group = userGroupMap[user];
var browser = userBrowserMap[user] || 'unknown';
// Find first event date
var funnelStartDay = null;
var funnelStartBrowser = null;
for (event in levelUserEventMap[level][user]) {
var day = levelUserEventMap[level][user][event];
if (!levelDayGroupEventMap[level]) levelDayGroupEventMap[level] = {};
if (!levelDayGroupEventMap[level][day]) levelDayGroupEventMap[level][day] = {};
if (!levelDayGroupEventMap[level][day][group]) levelDayGroupEventMap[level][day][group] = {};
if (!levelDayGroupEventMap[level][day][group][event]) levelDayGroupEventMap[level][day][group][event] = 0;
var day = levelUserEventMap[level][user][event]['day'];
var ls = levelUserEventMap[level][user][event]['ls'];
if (lsBrowserMap[ls]) {
browser = lsBrowserMap[ls];
}
if (!levelDayGroupBrowserEventMap[level]) levelDayGroupBrowserEventMap[level] = {};
if (!levelDayGroupBrowserEventMap[level][day]) levelDayGroupBrowserEventMap[level][day] = {};
if (!levelDayGroupBrowserEventMap[level][day][group]) levelDayGroupBrowserEventMap[level][day][group] = {};
if (!levelDayGroupBrowserEventMap[level][day][group][browser]) {
levelDayGroupBrowserEventMap[level][day][group][browser] = {};
}
if (!levelDayGroupBrowserEventMap[level][day][group][browser][event]) {
levelDayGroupBrowserEventMap[level][day][group][browser][event] = 0;
}
if (eventFunnel[0] === event) {
// First event gets attributed to current date
levelDayGroupEventMap[level][day][group][event]++;
levelDayGroupBrowserEventMap[level][day][group][browser][event]++;
funnelStartDay = day;
funnelStartBrowser = browser;
break;
}
}
if (funnelStartDay) {
if (!levelDayGroupEventMap[level][funnelStartDay][group]) {
levelDayGroupEventMap[level][funnelStartDay][group] = {};
}
// Add remaining funnel steps/events to first step's date
for (event in levelUserEventMap[level][user]) {
if (!levelDayGroupEventMap[level][funnelStartDay][group][event]) {
levelDayGroupEventMap[level][funnelStartDay][group][event] = 0;
if (!levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser]) {
levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser] = {};
}
if (!levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser][event]) {
levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser][event] = 0;
}
if (eventFunnel[0] !== event) {
levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser][event]++;
}
if (eventFunnel[0] !== event) levelDayGroupEventMap[level][funnelStartDay][group][event]++;
}
// Zero remaining funnel events
for (var i = 1; i < eventFunnel.length; i++) {
var event = eventFunnel[i];
if (!levelDayGroupEventMap[level][funnelStartDay][group][event]) {
levelDayGroupEventMap[level][funnelStartDay][group][event] = 0;
if (!levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser][event]) {
levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser][event] = 0;
}
if (!levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser][event]) {
levelDayGroupBrowserEventMap[level][funnelStartDay][group][funnelStartBrowser][event] = 0;
}
}
}
// Else no start event in this date range
}
}
// printjson(levelDayGroupEventMap);
// printjson(levelDayGroupBrowserEventMap);
log("Building results..");
var funnelData = [];
for (level in levelDayGroupEventMap) {
for (day in levelDayGroupEventMap[level]) {
for (group in levelDayGroupEventMap[level][day]) {
var started = 0;
var finished = 0;
for (event in levelDayGroupEventMap[level][day][group]) {
if (event === eventFunnel[0]) {
started = levelDayGroupEventMap[level][day][group][event];
}
else if (event === eventFunnel[eventFunnel.length - 1]) {
finished = levelDayGroupEventMap[level][day][group][event];
for (level in levelDayGroupBrowserEventMap) {
for (day in levelDayGroupBrowserEventMap[level]) {
for (group in levelDayGroupBrowserEventMap[level][day]) {
for (browser in levelDayGroupBrowserEventMap[level][day][group]) {
var started = 0;
var finished = 0;
for (event in levelDayGroupBrowserEventMap[level][day][group][browser]) {
if (event === eventFunnel[0]) {
started = levelDayGroupBrowserEventMap[level][day][group][browser][event];
}
else if (event === eventFunnel[eventFunnel.length - 1]) {
finished = levelDayGroupBrowserEventMap[level][day][group][browser][event];
}
}
funnelData.push({
level: level,
day: day,
group: group,
browser: browser,
started: started,
finished: finished
});
}
funnelData.push({
level: level,
day: day,
group: group,
started: started,
finished: finished
});
}
}
}
log("Sorting results..");
funnelData.sort(function (a,b) {
if (a.level !== b.level) {
return a.level < b.level ? -1 : 1;
@ -174,8 +232,79 @@ function getFunnelData(startDay, eventFunnel, testGroupFn, levelSlugs) {
else if (a.day !== b.day) {
return a.day < b.day ? -1 : 1;
}
else if (a.browser !== b.browser) {
return a.browser < b.browser ? -1 : 1;
}
return a.group < b.group ? -1 : 1;
});
return funnelData;
}
function printFunnelData(funnelData, printFn) {
log("Day\t\tGroup\t\tStarted\tFinished\tCompletion Rate");
var levelBrowserGroupCounts = {};
var levelGroupCounts = {};
var groupCounts = {};
for (var i = 0; i < funnelData.length; i++) {
var level = funnelData[i].level;
var day = funnelData[i].day;
var browser = funnelData[i].browser;
var group = funnelData[i].group;
var started = funnelData[i].started;
var finished = funnelData[i].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
printFn(day, level, browser, group, started, finished, rate);
if (!levelBrowserGroupCounts[level]) levelBrowserGroupCounts[level] = {};
if (!levelBrowserGroupCounts[level][browser]) levelBrowserGroupCounts[level][browser] = {};
if (!levelBrowserGroupCounts[level][browser][group]) {
levelBrowserGroupCounts[level][browser][group] = {started: 0, finished: 0};
}
levelBrowserGroupCounts[level][browser][group]['started'] += started;
levelBrowserGroupCounts[level][browser][group]['finished'] += finished;
if (!levelGroupCounts[level]) levelGroupCounts[level] = {};
if (!levelGroupCounts[level][group]) levelGroupCounts[level][group] = {started: 0, finished: 0};
levelGroupCounts[level][group]['started'] += started;
levelGroupCounts[level][group]['finished'] += finished;
if (!groupCounts[group]) groupCounts[group] = {started: 0, finished: 0};
groupCounts[group]['started'] += started;
groupCounts[group]['finished'] += finished;
}
log("");
log("Browser totals:");
for (level in levelBrowserGroupCounts) {
for (browser in levelBrowserGroupCounts[level]) {
for (group in levelBrowserGroupCounts[level][browser]) {
var started = levelBrowserGroupCounts[level][browser][group].started;
if (started < browserCountPrintThreshold) continue;
var finished = levelBrowserGroupCounts[level][browser][group].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
printFn(null, level, browser, group, started, finished, rate);
}
}
}
log("");
log("Level totals:");
for (level in levelGroupCounts) {
for (group in levelGroupCounts[level]) {
var started = levelGroupCounts[level][group].started;
var finished = levelGroupCounts[level][group].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
printFn(null, level, null, group, started, finished, rate);
}
}
log("");
log("Group totals:");
for (group in groupCounts) {
var started = groupCounts[group].started;
var finished = groupCounts[group].finished;
var rate = started > 0 ? finished / started * 100 : 0.0;
printFn(null, null, null, group, started, finished, rate);
}
}