/* global printjson */ /* global db */ // Insert per-day recurring revenue counts into analytics.perdays collection // Usage: // mongo <address>:<port>/<database> <script file> -u <username> -p <password> // TODO: Investigate school sales amount == zero. Manual input error? var scriptStartTime = new Date(); var analyticsStringCache = {}; var numDays = 40; var daysInMonth = 30; var startDay = new Date(); today = startDay.toISOString().substr(0, 10); startDay.setUTCDate(startDay.getUTCDate() - numDays); startDay = startDay.toISOString().substr(0, 10); log("Today is " + today); log("Start day is " + startDay); log("Getting recurring revenue counts..."); var recurringRevenueCounts = getRecurringRevenueCounts(startDay); // printjson(recurringRevenueCounts); log("Inserting recurring revenue counts..."); for (var event in recurringRevenueCounts) { for (var day in recurringRevenueCounts[event]) { if (today === day) continue; // Never save data for today because it's incomplete // print(event, day, recurringRevenueCounts[event][day]); insertEventCount(event, day, recurringRevenueCounts[event][day]); } } log("Script runtime: " + (new Date() - scriptStartTime)); function getRecurringRevenueCounts(startDay) { if (!startDay) return {}; var dailyRevenueCounts = {}; var day; var prepaidIDs = []; var prepaidDayAmountMap = {}; var startObj = objectIdWithTimestamp(ISODate(startDay + "T00:00:00.000Z")); var cursor = db.payments.find({_id: {$gte: startObj}}); while (cursor.hasNext()) { var doc = cursor.next(); if (doc.created) { day = new Date(doc.created).toISOString().substring(0, 10); } else { day = doc._id.getTimestamp().toISOString().substring(0, 10); } if (doc.service === 'ios' || doc.service === 'bitcoin') continue; if (!doc.amount || doc.amount <= 0) continue; if (doc.prepaidID) { if (prepaidDayAmountMap[doc.prepaidID.valueOf()]) { console.log("ERROR! prepaid", doc.prepaidID.valueOf(), "attached to multiple payments", doc._id.valueOf(), prepaidDayAmountMap[doc.prepaidID.valueOf()]); return {}; } prepaidDayAmountMap[doc.prepaidID.valueOf()] = {day: day, amount: doc.amount}; prepaidIDs.push(doc.prepaidID); } else if (doc.productID && doc.productID.indexOf('gems_') === 0) { if (!dailyRevenueCounts['DRR gems']) dailyRevenueCounts['DRR gems'] = {}; if (!dailyRevenueCounts['DRR gems'][day]) dailyRevenueCounts['DRR gems'][day] = 0; dailyRevenueCounts['DRR gems'][day] += doc.amount; } else if (doc.productID === 'custom' || doc.service === 'external' || doc.service === 'invoice') { if (!dailyRevenueCounts['DRR school sales']) dailyRevenueCounts['DRR school sales'] = {}; if (!dailyRevenueCounts['DRR school sales'][day]) dailyRevenueCounts['DRR school sales'][day] = 0; dailyRevenueCounts['DRR school sales'][day] += doc.amount; } else if (doc.service === 'stripe' && doc.gems === 42000) { if (!dailyRevenueCounts['DRR yearly subs']) dailyRevenueCounts['DRR yearly subs'] = {}; if (!dailyRevenueCounts['DRR yearly subs'][day]) dailyRevenueCounts['DRR yearly subs'][day] = 0; dailyRevenueCounts['DRR yearly subs'][day] += doc.amount; } else if (doc.service === 'stripe') { // Catches prepaids, and assumes all are type terminal_subscription if (!dailyRevenueCounts['DRR monthly subs']) dailyRevenueCounts['DRR monthly subs'] = {}; if (!dailyRevenueCounts['DRR monthly subs'][day]) dailyRevenueCounts['DRR monthly subs'][day] = 0; dailyRevenueCounts['DRR monthly subs'][day] += doc.amount; } else if (doc.service === 'paypal') { if (!dailyRevenueCounts['DRR monthly subs']) dailyRevenueCounts['DRR monthly subs'] = {}; if (!dailyRevenueCounts['DRR monthly subs'][day]) dailyRevenueCounts['DRR monthly subs'][day] = 0; dailyRevenueCounts['DRR monthly subs'][day] += doc.amount; } // else { // // printjson(doc); // // print(doc.service, doc.amount, doc.description, JSON.stringify(doc.stripe)); // } } // Add revenue from prepaids connected to payments cursor = db.prepaids.find({_id: {$in: prepaidIDs}}); while (cursor.hasNext()) { doc = cursor.next(); if (prepaidDayAmountMap[doc._id.valueOf()]) { day = prepaidDayAmountMap[doc._id.valueOf()].day; var amount = prepaidDayAmountMap[doc._id.valueOf()].amount; if (doc.type === 'course' || doc.type === 'terminal_subscription') { var revenueType = doc.type === 'course' ? 'DRR school sales' : 'DRR monthly subs'; if (!dailyRevenueCounts[revenueType]) dailyRevenueCounts[revenueType] = {}; if (!dailyRevenueCounts[revenueType][day]) dailyRevenueCounts[revenueType][day] = 0; dailyRevenueCounts[revenueType][day] += amount; } } } return dailyRevenueCounts; } // *** Helper functions *** function slugify(text) // https://gist.github.com/mathewbyrne/1280286 { return text.toString().toLowerCase() .replace(/\s+/g, '-') // Replace spaces with - .replace(/[^\w\-]+/g, '') // Remove all non-word chars .replace(/\-\-+/g, '-') // Replace multiple - with single - .replace(/^-+/, '') // Trim - from start of text .replace(/-+$/, ''); // Trim - from end of text } function log(str) { print(new Date().toISOString() + " " + str); } function objectIdWithTimestamp(timestamp) { // Convert string date to Date object (otherwise assume timestamp is a date) if (typeof(timestamp) == 'string') timestamp = new Date(timestamp); // Convert date object to hex seconds since Unix epoch var hexSeconds = Math.floor(timestamp/1000).toString(16); // Create an ObjectId with that hex timestamp var constructedObjectId = ObjectId(hexSeconds + "0000000000000000"); return constructedObjectId } function getAnalyticsString(str) { if (analyticsStringCache[str]) return analyticsStringCache[str]; // Find existing string var doc = db['analytics.strings'].findOne({v: str}); if (doc) { analyticsStringCache[str] = doc._id; return analyticsStringCache[str]; } // Insert string // http://docs.mongodb.org/manual/tutorial/create-an-auto-incrementing-field/#auto-increment-optimistic-loop doc = {v: str}; while (true) { var cursor = db['analytics.strings'].find({}, {_id: 1}).sort({_id: -1}).limit(1); var seq = cursor.hasNext() ? cursor.next()._id + 1 : 1; doc._id = seq; var results = db['analytics.strings'].insert(doc); if (results.hasWriteError()) { if ( results.writeError.code == 11000 /* dup key */ ) continue; else throw new Error("ERROR: Unexpected error inserting data: " + tojson(results)); } break; } // Find new string entry doc = db['analytics.strings'].findOne({v: str}); if (doc) { analyticsStringCache[str] = doc._id; return analyticsStringCache[str]; } throw new Error("ERROR: Did not find analytics.strings insert for: " + str); } function insertEventCount(event, day, count) { // analytics.perdays schema in server/analytics/AnalyticsPeryDay.coffee day = day.replace(/-/g, ''); var results; var eventID = getAnalyticsString(event); var filterID = getAnalyticsString('all'); var queryParams = {$and: [{d: day}, {e: eventID}, {f: filterID}]}; var doc = db['analytics.perdays'].findOne(queryParams); if (doc && doc.c === count) return; if (doc && doc.c !== count) { // Update existing count, assume new one is more accurate // log("Updating count in db for " + day + " " + event + " " + doc.c + " => " + count); results = db['analytics.perdays'].update(queryParams, {$set: {c: count}}); if (results.nMatched !== 1 && results.nModified !== 1) { log("ERROR: update event count failed"); printjson(results); } } else { var insertDoc = {d: day, e: eventID, f: filterID, c: count}; results = db['analytics.perdays'].insert(insertDoc); if (results.nInserted !== 1) { log("ERROR: insert event failed"); printjson(results); printjson(insertDoc); } // else { // log("Added " + day + " " + event + " " + count); // } } }