# Calculate gem prompt A/B test results # TODO: Why is no-prompt group 50% larger? import sys from mixpanel import Mixpanel try: import json except ImportError: import simplejson as json # NOTE: mixpanel dates are by day and inclusive # E.g. '2014-12-08' is any date that day, up to 2014-12-09 12am if __name__ == '__main__': if not len(sys.argv) is 3: print "Script format: <script> <api_key> <api_secret>" else: api_key = sys.argv[1] api_secret = sys.argv[2] api = Mixpanel( api_key = api_key, api_secret = api_secret ) startDate = '2015-01-15' startDate = '2014-11-25' endDate = '2015-02-11' print("Requesting data for {0} to {1}".format(startDate, endDate)) data = api.request(['export'], { 'event' : ['Started purchase', 'Finished gem purchase'], 'from_date' : startDate, 'to_date' : endDate }) userProgressionGroupA = {} userProgressionGroupB = {} lines = data.split('\n') print "Received %d entries" % len(lines) for line in lines: try: if len(line) is 0: continue eventData = json.loads(line) eventName = eventData['event'] properties = eventData['properties'] if not eventName in ['Started purchase', 'Finished gem purchase']: print 'Unexpected event ' + eventName break if 'distinct_id' in properties and 'gemPromptGroup' in properties: userID = properties['distinct_id'] if properties['gemPromptGroup'] == 'prompt': if not userID in userProgressionGroupA: userProgressionGroupA[userID] = { 'Started purchase': 0, 'Finished gem purchase': 0 } userProgressionGroupA[userID][eventName] += 1 elif properties['gemPromptGroup'] == 'no-prompt': if not userID in userProgressionGroupB: userProgressionGroupB[userID] = { 'Started purchase': 0, 'Finished gem purchase': 0 } userProgressionGroupB[userID][eventName] += 1 else: print "Unexpected group:", properties['gemPromptGroup'] print properties print line break except: print "Unexpected error:", sys.exc_info()[0] print line break try: started = converted = 0 startedGroupA = convertedGroupA = 0 startedGroupB = convertedGroupB = 0 # Group A print("Processing Group A") for key, item in userProgressionGroupA.iteritems(): if item['Finished gem purchase'] > 0: converted += 1 convertedGroupA += 1 # TODO: is our distinct_id correct? We hit this at least once. # if item['Finished gem purchase'] > 1: # print "User multiple subcription purchases?" # print item elif item['Started purchase'] > 0: started += 1 startedGroupA += 1 else: print "User without any hits?" print item break # Group B print("Processing Group B") for key, item in userProgressionGroupB.iteritems(): if item['Finished gem purchase'] > 0: converted += 1 convertedGroupB += 1 elif item['Started purchase'] > 0: started += 1 startedGroupB += 1 else: print "User without any hits?" print item break print("Overall") print("started {0} converted {1} rate {2}%".format(started, converted, float(converted) / started * 100)) print("Group prompt") print("startedGroupA {0} convertedGroupA {1} rate {2}%".format(startedGroupA, convertedGroupA, float(convertedGroupA) / startedGroupA * 100)) print("Group no-prompt") print("startedGroupB {0} convertedGroupB {1} rate {2}%".format(startedGroupB, convertedGroupB, float(convertedGroupB) / startedGroupB * 100)) except: print "Unexpected error:", sys.exc_info()[0]