From c5977c00fe25769eb2a4ab2c1db3121025404f75 Mon Sep 17 00:00:00 2001 From: Matt Lott Date: Wed, 14 Jan 2015 17:51:04 -0800 Subject: [PATCH] Add shortened fields to analytics.log.event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’ll remove the old long fields after we’ve got enough data to switch over our analytics queries without complication. --- app/core/Tracker.coffee | 33 ++++++---- app/lib/scripts/ScriptManager.coffee | 8 +-- app/schemas/models/analytics_log_event.coffee | 5 ++ server/analytics/AnalyticsLogEvent.coffee | 11 +++- .../analytics_log_event_handler.coffee | 64 +++++++++++++++++++ server/lib/utils.coffee | 23 +++++++ 6 files changed, 125 insertions(+), 19 deletions(-) diff --git a/app/core/Tracker.coffee b/app/core/Tracker.coffee index 45f012f6a..5e40bacfb 100644 --- a/app/core/Tracker.coffee +++ b/app/core/Tracker.coffee @@ -1,5 +1,5 @@ {me} = require 'core/auth' -AnalyticsLogEvent = require 'models/AnalyticsLogEvent' +SuperModel = require 'models/SuperModel' debugAnalytics = false @@ -10,6 +10,7 @@ module.exports = class Tracker window.tracker = @ @isProduction = document.location.href.search('codecombat.com') isnt -1 @identify() + @supermodel = new SuperModel() identify: (traits) -> console.log 'Would identify', traits if debugAnalytics @@ -61,19 +62,7 @@ module.exports = class Tracker # https://segment.com/docs/integrations/mixpanel/ properties = properties or {} - # Log internally - # Skipping heavily logged actions we don't use internally - unless action in ['Simulator Result', 'Started Level Load', 'Finished Level Load'] - # Trimming properties we don't use internally - # TODO: delete internalProperites.level for 'Saw Victory' after 2/8/15. Should be using levelID instead. - internalProperties = _.cloneDeep properties - if action in ['Clicked Level', 'Inventory Play', 'Heard Sprite', 'Started Level', 'Saw Victory', 'Click Play', 'Choose Inventory'] - delete internalProperties.category - delete internalProperties.label - - console.log 'Tracking internal analytics event:', action, internalProperties, includeIntegrations if debugAnalytics - event = new AnalyticsLogEvent event: action, properties: internalProperties - event.save() + @trackEventInternal action, _.cloneDeep properties console.log 'Would track analytics event:', action, properties, includeIntegrations if debugAnalytics return unless me and @isProduction and analytics? and not me.isAdmin() @@ -85,6 +74,22 @@ module.exports = class Tracker context.integrations[integration] = true analytics?.track action, properties, context + trackEventInternal: (event, properties) => + # Skipping heavily logged actions we don't use internally + unless event in ['Simulator Result', 'Started Level Load', 'Finished Level Load'] + # Trimming properties we don't use internally + # TODO: delete internalProperites.level for 'Saw Victory' after 2/8/15. Should be using levelID instead. + if event in ['Clicked Level', 'Inventory Play', 'Heard Sprite', 'Started Level', 'Saw Victory', 'Click Play', 'Choose Inventory', 'Loaded World Map', 'Homepage Loaded'] + delete properties.category + delete properties.label + + console.log 'Tracking internal analytics event:', event, properties if debugAnalytics + request = @supermodel.addRequestResource 'log_event', { + url: '/db/analytics_log_event/-/log_event' + data: {event: event, properties: properties} + method: 'POST' + }, 0 + request.load() trackTiming: (duration, category, variable, label, samplePercentage=5) -> # https://developers.google.com/analytics/devguides/collection/gajs/gaTrackingTiming diff --git a/app/lib/scripts/ScriptManager.coffee b/app/lib/scripts/ScriptManager.coffee index 9d6810203..c5a42dda9 100644 --- a/app/lib/scripts/ScriptManager.coffee +++ b/app/lib/scripts/ScriptManager.coffee @@ -220,8 +220,8 @@ module.exports = ScriptManager = class ScriptManager extends CocoClass @notifyScriptStateChanged() @scriptInProgress = true @currentTimeouts = [] - scriptLabel = "#{@levelID}: #{nextNoteGroup.scriptID} - #{nextNoteGroup.name}" - application.tracker?.trackEvent 'Script Started', {label: scriptLabel}, ['Google Analytics'] + scriptLabel = "#{nextNoteGroup.scriptID} - #{nextNoteGroup.name}" + application.tracker?.trackEvent 'Script Started', {levelID: @levelID, label: scriptLabel}, ['Google Analytics'] console.debug "SCRIPT: Starting note group '#{nextNoteGroup.name}'" if @debugScripts for module in nextNoteGroup.modules @processNote(note, nextNoteGroup) for note in module.startNotes() @@ -283,8 +283,8 @@ module.exports = ScriptManager = class ScriptManager extends CocoClass return if @ending # kill infinite loops right here @ending = true return unless @currentNoteGroup? - scriptLabel = "#{@levelID}: #{@currentNoteGroup.scriptID} - #{@currentNoteGroup.name}" - application.tracker?.trackEvent 'Script Ended', {label: scriptLabel}, ['Google Analytics'] + scriptLabel = "#{@currentNoteGroup.scriptID} - #{@currentNoteGroup.name}" + application.tracker?.trackEvent 'Script Ended', {levelID: @levelID, label: scriptLabel}, ['Google Analytics'] console.debug "SCRIPT: Ending note group '#{@currentNoteGroup.name}'" if @debugScripts clearTimeout(timeout) for timeout in @currentTimeouts for module in @currentNoteGroup.modules diff --git a/app/schemas/models/analytics_log_event.coffee b/app/schemas/models/analytics_log_event.coffee index 1468d0ecc..d3601a731 100644 --- a/app/schemas/models/analytics_log_event.coffee +++ b/app/schemas/models/analytics_log_event.coffee @@ -6,6 +6,11 @@ AnalyticsLogEventSchema = c.object { } _.extend AnalyticsLogEventSchema.properties, + u: c.objectId(links: [{rel: 'extra', href: '/db/user/{($)}'}]) + e: {type: 'integer'} + p: {type: 'object'} + + # TODO: Remove these legacy properties after we stop querying for them (probably 30 days, ~2/16/15) user: c.objectId(links: [{rel: 'extra', href: '/db/user/{($)}'}]) event: {type: 'string'} properties: {type: 'object'} diff --git a/server/analytics/AnalyticsLogEvent.coffee b/server/analytics/AnalyticsLogEvent.coffee index 62364016e..18a146346 100644 --- a/server/analytics/AnalyticsLogEvent.coffee +++ b/server/analytics/AnalyticsLogEvent.coffee @@ -1,7 +1,16 @@ mongoose = require 'mongoose' plugins = require '../plugins/plugins' -AnalyticsLogEventSchema = new mongoose.Schema({}, {strict: false}) +AnalyticsLogEventSchema = new mongoose.Schema({ + u: mongoose.Schema.Types.ObjectId + e: Number # event analytics.string ID + p: mongoose.Schema.Types.Mixed + + # TODO: Remove these legacy properties after we stop querying for them (probably 30 days, ~2/16/15) + user: mongoose.Schema.Types.ObjectId + event: String + properties: mongoose.Schema.Types.Mixed +}, {strict: false}) AnalyticsLogEventSchema.index({event: 1, _id: 1}) module.exports = AnalyticsLogEvent = mongoose.model('analytics.log.event', AnalyticsLogEventSchema) diff --git a/server/analytics/analytics_log_event_handler.coffee b/server/analytics/analytics_log_event_handler.coffee index 1ef08a8fc..0d8c63298 100644 --- a/server/analytics/analytics_log_event_handler.coffee +++ b/server/analytics/analytics_log_event_handler.coffee @@ -9,6 +9,8 @@ class AnalyticsLogEventHandler extends Handler modelClass: AnalyticsLogEvent jsonSchema: require '../../app/schemas/models/analytics_log_event' editableProperties: [ + 'e' + 'p' 'event' 'properties' ] @@ -18,15 +20,77 @@ class AnalyticsLogEventHandler extends Handler makeNewInstance: (req) -> instance = super(req) + instance.set('u', req.user._id) + # TODO: Remove 'user' after we stop querying for it (probably 30 days, ~2/16/15) instance.set('user', req.user._id) instance getByRelationship: (req, res, args...) -> + return @logEvent(req, res) if args[1] is 'log_event' # TODO: Remove these APIs # return @getLevelCompletionsBySlug(req, res) if args[1] is 'level_completions' # return @getCampaignCompletionsBySlug(req, res) if args[1] is 'campaign_completions' super(arguments...) + logEvent: (req, res) -> + # Converts strings to string IDs where possible, and logs the event + user = req.user._id + event = req.query.event or req.body.event + properties = req.query.properties or req.body.properties + @sendSuccess res # Return request immediately + + saveDoc = (eventID, slimProperties) -> + doc = new AnalyticsLogEvent + u: user + e: eventID + p: slimProperties + # TODO: Remove these legacy properties after we stop querying for them (probably 30 days, ~2/16/15) + user: user + event: event + properties: properties + doc.save() + + utils.getAnalyticsStringID event, (eventID) -> + if eventID > 0 + # TODO: properties slimming is pretty ugly + slimProperties = _.cloneDeep properties + if event is 'Saw Victory' + delete slimProperties.level + if slimProperties.levelID? + # levelID: string => l: string ID + utils.getAnalyticsStringID slimProperties.levelID, (levelStringID) -> + if levelStringID > 0 + delete slimProperties.levelID + slimProperties.l = levelStringID + saveDoc eventID, slimProperties + return + else if event is 'Started Level' + if slimProperties.levelID? + # levelID: string => l: string ID + utils.getAnalyticsStringID slimProperties.levelID, (levelStringID) -> + if levelStringID > 0 + delete slimProperties.levelID + slimProperties.l = levelStringID + saveDoc eventID, slimProperties + return + else if event in ['Script Started', 'Script Ended'] + if slimProperties.levelID? + # levelID: string => l: string ID + # label: string => lb: string ID + utils.getAnalyticsStringID slimProperties.levelID, (levelStringID) -> + if levelStringID > 0 + delete slimProperties.levelID + slimProperties.l = levelStringID + utils.getAnalyticsStringID slimProperties.label, (labelStringID) -> + if labelStringID > 0 + delete slimProperties.label + slimProperties.lb = labelStringID + saveDoc eventID, slimProperties + return + saveDoc eventID, slimProperties + else + log.warn "Unable to get analytics string ID for " + event + getLevelCompletionsBySlug: (req, res) -> # Returns an array of per-day level starts and finishes # Parameters: diff --git a/server/lib/utils.coffee b/server/lib/utils.coffee index 18aab7c79..45408f558 100644 --- a/server/lib/utils.coffee +++ b/server/lib/utils.coffee @@ -1,3 +1,4 @@ +AnalyticsString = require '../analytics/AnalyticsString' mongoose = require 'mongoose' module.exports = @@ -13,3 +14,25 @@ module.exports = hexSeconds = Math.floor(timestamp/1000).toString(16) # Create an ObjectId with that hex timestamp mongoose.Types.ObjectId(hexSeconds + "0000000000000000") + getAnalyticsStringID: (str, callback) -> + @analyticsStringCache ?= {} + return callback @analyticsStringCache[str] if @analyticsStringCache[str] + + insertString = => + # http://docs.mongodb.org/manual/tutorial/create-an-auto-incrementing-field/#auto-increment-optimistic-loop + AnalyticsString.find({}, {_id: 1}).sort({_id: -1}).limit(1).exec (err, documents) => + if err? then return callback -1 + seq = if documents.length > 0 then documents[0]._id + 1 else 1 + doc = new AnalyticsString _id: seq, v: str + doc.save (err) => + if err? then return callback -1 + @analyticsStringCache[str] = seq + callback seq + + # Find existing string + AnalyticsString.findOne(v: str).exec (err, document) => + if err? then return callback -1 + if document + @analyticsStringCache[str] = document._id + return callback @analyticsStringCache[str] + insertString()