From ed7d7b0412f5411d71d39666c423691133b7d587 Mon Sep 17 00:00:00 2001
From: Matt Lott <mattlott@live.com>
Date: Wed, 11 Feb 2015 13:57:28 -0800
Subject: [PATCH] Mixpanel gem prompt a/b test script

Investigating some weirdness where no-prompt group is 50% larger than
prompt group.  True in Mixpanel and in our internal analytics.
---
 scripts/analytics/mixpanelABGemPrompt.py      | 122 ++++++++++++++++++
 .../mongodb/queries/abGemPromptGroup.js       |   3 +
 2 files changed, 125 insertions(+)
 create mode 100644 scripts/analytics/mixpanelABGemPrompt.py

diff --git a/scripts/analytics/mixpanelABGemPrompt.py b/scripts/analytics/mixpanelABGemPrompt.py
new file mode 100644
index 000000000..43682f4ea
--- /dev/null
+++ b/scripts/analytics/mixpanelABGemPrompt.py
@@ -0,0 +1,122 @@
+# Calculate gem prompt A/B test results
+
+# TODO: Why is no-prompt group 50% larger?
+
+import sys
+from mixpanel import Mixpanel
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+# NOTE: mixpanel dates are by day and inclusive
+# E.g. '2014-12-08' is any date that day, up to 2014-12-09 12am
+
+if __name__ == '__main__':
+    if not len(sys.argv) is 3:
+        print "Script format: <script> <api_key> <api_secret>"
+    else:
+        api_key = sys.argv[1]
+        api_secret = sys.argv[2]
+        api = Mixpanel(
+            api_key = api_key,
+            api_secret = api_secret
+        )
+
+        startDate = '2015-01-15'
+        startDate = '2014-11-25'
+        endDate = '2015-02-11'
+
+        print("Requesting data for {0} to {1}".format(startDate, endDate))
+        data = api.request(['export'], {
+            'event' : ['Started purchase', 'Finished gem purchase'],
+            'from_date' : startDate,
+            'to_date' : endDate
+        })
+
+        userProgressionGroupA = {}
+        userProgressionGroupB = {}
+
+        lines = data.split('\n')
+        print "Received %d entries" % len(lines)
+        for line in lines:
+            try:
+                if len(line) is 0: continue
+                eventData = json.loads(line)
+                eventName = eventData['event']
+                properties = eventData['properties']
+                if not eventName in ['Started purchase', 'Finished gem purchase']:
+                    print 'Unexpected event ' + eventName
+                    break
+                if 'distinct_id' in properties and 'gemPromptGroup' in properties:
+                    userID = properties['distinct_id']
+                    if properties['gemPromptGroup'] == 'prompt':
+                        if not userID in userProgressionGroupA:
+                            userProgressionGroupA[userID] = {
+                                'Started purchase': 0,
+                                'Finished gem purchase': 0
+                            }
+                        userProgressionGroupA[userID][eventName] += 1
+                    elif properties['gemPromptGroup'] == 'no-prompt':
+                        if not userID in userProgressionGroupB:
+                            userProgressionGroupB[userID] = {
+                                'Started purchase': 0,
+                                'Finished gem purchase': 0
+                            }
+                        userProgressionGroupB[userID][eventName] += 1
+                    else:
+                        print "Unexpected group:", properties['gemPromptGroup']
+                        print properties
+                        print line
+                        break
+            except:
+                print "Unexpected error:", sys.exc_info()[0]
+                print line
+                break
+
+        try:
+            started = converted = 0
+            startedGroupA = convertedGroupA = 0
+            startedGroupB = convertedGroupB = 0
+
+            # Group A
+            print("Processing Group A")
+            for key, item in userProgressionGroupA.iteritems():
+                if item['Finished gem purchase'] > 0:
+                    converted += 1
+                    convertedGroupA += 1
+                    # TODO: is our distinct_id correct?  We hit this at least once.
+                    # if item['Finished gem purchase'] > 1:
+                    #     print "User multiple subcription purchases?"
+                    #     print item
+                elif item['Started purchase'] > 0:
+                    started += 1
+                    startedGroupA += 1
+                else:
+                    print "User without any hits?"
+                    print item
+                    break
+
+            # Group B
+            print("Processing Group B")
+            for key, item in userProgressionGroupB.iteritems():
+                if item['Finished gem purchase'] > 0:
+                    converted += 1
+                    convertedGroupB += 1
+                elif item['Started purchase'] > 0:
+                    started += 1
+                    startedGroupB += 1
+                else:
+                    print "User without any hits?"
+                    print item
+                    break
+
+            print("Overall")
+            print("started {0} converted {1} rate {2}%".format(started, converted, float(converted) / started * 100))
+            print("Group prompt")
+            print("startedGroupA {0} convertedGroupA {1} rate {2}%".format(startedGroupA, convertedGroupA, float(convertedGroupA) / startedGroupA * 100))
+            print("Group no-prompt")
+            print("startedGroupB {0} convertedGroupB {1} rate {2}%".format(startedGroupB, convertedGroupB, float(convertedGroupB) / startedGroupB * 100))
+        except:
+            print "Unexpected error:", sys.exc_info()[0]
diff --git a/scripts/analytics/mongodb/queries/abGemPromptGroup.js b/scripts/analytics/mongodb/queries/abGemPromptGroup.js
index cdc0a3a34..79cb78364 100644
--- a/scripts/analytics/mongodb/queries/abGemPromptGroup.js
+++ b/scripts/analytics/mongodb/queries/abGemPromptGroup.js
@@ -4,11 +4,14 @@
 // Usage:
 // mongo <address>:<port>/<database> <script file> -u <username> -p <password>
 
+// TODO: Why is no-prompt group 50% larger?
+
 load('abTestHelpers.js');
 
 var scriptStartTime = new Date();
 try {
   var startDay = '2014-11-24'
+  // startDay = '2015-01-15'
   log("Today is " + new Date().toISOString().substr(0, 10));
   log("Start day is " + startDay);