Improvements for recalculating earned achievements, recreating earned achievements that should exist, and updating proportionalTo achievements like simulatedBy that don't get updated with a post-save hook.

scripts/recreateEarnedAchievements.coffee

@@ -0,0 +1,91 @@
+database = require '../server/commons/database'
+mongoose = require 'mongoose'
+log = require 'winston'
+async = require 'async'
+
+### SET UP ###
+do (setupLodash = this) ->
+  GLOBAL._ = require 'lodash'
+  _.str = require 'underscore.string'
+  _.mixin _.str.exports()
+
+database.connect()
+
+LocalMongo = require '../app/lib/LocalMongo'
+User = require '../server/users/User'
+EarnedAchievement = require '../server/achievements/EarnedAchievement'
+Achievement = require '../server/achievements/Achievement'
+Achievement.loadAchievements (achievementCategories) ->
+  # Really, it's just the 'users' category, since we don't keep all the LevelSession achievements in memory, rather letting the clients make those.
+  userAchievements = achievementCategories.users
+  console.log 'There are', userAchievements.length, 'user achievements.'
+
+  # 0. Stream all the non-anonymous users.
+  # 1. Adapt this logic below to fetch the earned achievement to see if it exists.
+  # 2. If it doesn't exist, make it and add to total.
+  # 3. Keep other totals, too, for interesting stats.
+  # 4. Print out how long it's going to take.
+  # 5. process.exit()
+
+  t0 = new Date().getTime()
+  total = 100000
+  #testUsers = ['livelily+test31@gmail.com', 'livelily+test37@gmail.com']
+  if testUsers?
+    userQuery = emailLower: {$in: testUsers}
+  else
+    userQuery = anonymous: false
+  User.count userQuery, (err, count) -> total = count
+
+  onFinished = ->
+    t1 = new Date().getTime()
+    runningTime = ((t1-t0)/1000/60/60).toFixed(2)
+    console.log "we finished in #{runningTime} hours"
+    process.exit()
+
+  # Fetch every single user. This tends to get big so do it in a streaming fashion.
+  userStream = User.find(userQuery).sort('_id').stream()
+  streamFinished = false
+  usersTotal = 0
+  usersFinished = 0
+  totalAchievementsExisting = 0
+  totalAchievementsCreated = 0
+  numberRunning = 0
+  doneWithUser = ->
+    ++usersFinished
+    numberRunning -= 1
+    userStream.resume()
+    onFinished?() if streamFinished and usersFinished is usersTotal
+
+  userStream.on 'error', (err) -> log.error err
+  userStream.on 'close', -> streamFinished = true
+  userStream.on 'data', (user) ->
+    ++usersTotal
+    numberRunning += 1
+    userStream.pause() if numberRunning > 20
+    userID = user.get('_id').toHexString()
+    userObject = user.toObject()
+
+    # Fetch all of a user's earned achievements
+    EarnedAchievement.find {user: userID}, (err, alreadyEarnedAchievements) ->
+      log.error err if err
+
+      achievementsExisting = 0
+      achievementsCreated = 0
+      for achievement in userAchievements
+        #console.log "Testing", achievement.get('name'), achievement.get('_id') if testUsers?
+        shouldBeAchieved = LocalMongo.matchesQuery userObject, achievement.get('query')
+        continue unless shouldBeAchieved  # Could delete existing ones that shouldn't be achieved if we wanted.
+        earnedAchievement = _.find(alreadyEarnedAchievements, (ea) -> ea.get('user') is userID and ea.get('achievement') is achievement.get('_id').toHexString())
+        if earnedAchievement
+          #console.log "... already earned #{achievement.get('name')} #{achievement.get('_id')} for user: #{user.get('name')} #{user.get('_id')}" if testUsers?
+          ++achievementsExisting
+          continue
+        #console.log "Making an achievement: #{achievement.get('name')} #{achievement.get('_id')} for user: #{user.get('name')} #{user.get('_id')}" if testUsers?
+        ++achievementsCreated
+        EarnedAchievement.createForAchievement achievement, user
+      
+      totalAchievementsExisting += achievementsExisting
+      totalAchievementsCreated += achievementsCreated
+      pctDone = (100 * usersFinished / total).toFixed(2)
+      console.log "Created #{achievementsCreated}, existing #{achievementsExisting} EarnedAchievements for #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%, totals #{totalAchievementsExisting} existing, #{totalAchievementsCreated} created)"
+      doneWithUser()

server/achievements/Achievement.coffee

@@ -47,7 +47,7 @@ AchievementSchema.methods.getExpFunction = ->
   return utils.functionCreators[func.kind](func.parameters) if func.kind of utils.functionCreators
 
 AchievementSchema.statics.jsonschema = jsonschema
-AchievementSchema.statics.earnedAchievements = {}
+AchievementSchema.statics.achievementCategories = {}
 
 # Reloads all achievements into memory.
 # TODO might want to tweak this to only load new achievements
@@ -58,15 +58,20 @@ AchievementSchema.statics.loadAchievements = (done) ->
   query.exec (err, docs) ->
     _.each docs, (achievement) ->
       category = achievement.get 'collection'
-      AchievementSchema.statics.earnedAchievements[category] = [] unless category of AchievementSchema.statics.earnedAchievements
-      AchievementSchema.statics.earnedAchievements[category].push achievement
-    done?(AchievementSchema.statics.earnedAchievements)
+      AchievementSchema.statics.achievementCategories[category] ?= []
+      if _.find AchievementSchema.statics.achievementCategories[category], ((a) -> a.get('_id').toHexString() is achievement.get('_id').toHexString())
+        log.warn "Uh oh, we tried to add another copy of the same achievement #{achievement.get('_id')} #{achievement.get('name')} to the #{category} achievement list..."
+      else
+        AchievementSchema.statics.achievementCategories[category].push achievement
+      unless achievement.get('query')
+        log.error "Uh oh, there is an achievement with an empty query: #{achievement}"
+    done?(AchievementSchema.statics.achievementCategories)
 
 AchievementSchema.statics.getLoadedAchievements = ->
-  AchievementSchema.statics.earnedAchievements
+  AchievementSchema.statics.achievementCategories
 
 AchievementSchema.statics.resetAchievements = ->
-  delete AchievementSchema.statics.earnedAchievements[category] for category of AchievementSchema.statics.earnedAchievements
+  delete AchievementSchema.statics.achievementCategories[category] for category of AchievementSchema.statics.achievementCategories
 
 # Queries are stored as JSON strings, objectify them upon loading
 AchievementSchema.post 'init', (doc) -> doc.objectifyQuery()
@@ -76,6 +81,7 @@ AchievementSchema.pre 'save', (next) ->
   next()
 
 # Reload achievements upon save
+# This is going to basically not work when there is more than one application server, right?
 AchievementSchema.post 'save', -> @constructor.loadAchievements()
 
 AchievementSchema.plugin(plugins.NamedPlugin)

server/achievements/EarnedAchievement.coffee

@@ -50,29 +50,36 @@ EarnedAchievementSchema.statics.createForAchievement = (achievement, doc, origin
     proportionalTo = achievement.get 'proportionalTo'
     docObj = doc.toObject()
     newAmount = util.getByPath(docObj, proportionalTo) or 0
-    if previouslyEarnedAchievement
-      originalAmount = previouslyEarnedAchievement.get('achievedAmount') or 0
+    updateEarnedAchievement = (originalAmount) ->
+      #console.log 'original amount is', originalAmount, 'and new amount is', newAmount, 'for', proportionalTo, 'with doc', docObj, 'and previously earned achievement amount', previouslyEarnedAchievement?.get('achievedAmount'), 'because we had originalDocObj', originalDocObj
+
+      if originalAmount isnt newAmount
+        expFunction = achievement.getExpFunction()
+        earned.notified = false
+        earned.achievedAmount = newAmount
+        #console.log 'earnedPoints is', (expFunction(newAmount) - expFunction(originalAmount)) * pointWorth, 'was', earned.earnedPoints, earned.previouslyAchievedAmount, 'got exp function for new amount', newAmount, expFunction(newAmount), 'for original amount', originalAmount, expFunction(originalAmount), 'with point worth', pointWorth
+        earnedPoints = earned.earnedPoints = (expFunction(newAmount) - expFunction(originalAmount)) * pointWorth
+        earnedGems = earned.earnedGems = (expFunction(newAmount) - expFunction(originalAmount)) * gemWorth
+        earned.previouslyAchievedAmount = originalAmount
+        EarnedAchievement.update {achievement: earned.achievement, user: earned.user}, earned, {upsert: true}, (err) ->
+          return log.error err if err?
+
+        wrapUp(new EarnedAchievement(earned))
+      else
+        done?()
+
+    if proportionalTo is 'simulatedBy' and newAmount > 0 and not previouslyEarnedAchievement and Math.random() < 0.1
+      # Because things like simulatedBy get updated with $inc and not the post-save plugin hook,
+      # we (infrequently) fetch the previously earned achievement so we can really update.
+      EarnedAchievement.findOne {user: earned.user, achievement: earned.achievement}, (err, previouslyEarnedAchievement) ->
+        log.error err if err?
+        updateEarnedAchievement previouslyEarnedAchievement?.get('achievedAmount') or 0
+    else if previouslyEarnedAchievement
+      updateEarnedAchievement previouslyEarnedAchievement.get('achievedAmount') or 0
     else if originalDocObj  # This branch could get buggy if unchangedCopy tracking isn't working.
-      originalAmount = util.getByPath(originalDocObj, proportionalTo) or 0
+      updateEarnedAchievement util.getByPath(originalDocObj, proportionalTo) or 0
     else
-      originalAmount = 0
-    #console.log 'original amount is', originalAmount, 'and new amount is', newAmount, 'for', proportionalTo, 'with doc', docObj, 'and previously earned achievement amount', previouslyEarnedAchievement?.get('achievedAmount'), 'because we had originalDocObj', originalDocObj
-
-    if originalAmount isnt newAmount
-      expFunction = achievement.getExpFunction()
-      earned.notified = false
-      earned.achievedAmount = newAmount
-      #console.log 'earnedPoints is', (expFunction(newAmount) - expFunction(originalAmount)) * pointWorth, 'was', earned.earnedPoints, earned.previouslyAchievedAmount, 'got exp function for new amount', newAmount, expFunction(newAmount), 'for original amount', originalAmount, expFunction(originalAmount), 'with point worth', pointWorth
-      earnedPoints = earned.earnedPoints = (expFunction(newAmount) - expFunction(originalAmount)) * pointWorth
-      earnedGems = earned.earnedGems = (expFunction(newAmount) - expFunction(originalAmount)) * gemWorth
-      earned.previouslyAchievedAmount = originalAmount
-      EarnedAchievement.update {achievement: earned.achievement, user: earned.user}, earned, {upsert: true}, (err) ->
-        return log.debug err if err?
-
-      #log.debug earnedPoints
-      wrapUp(new EarnedAchievement(earned))
-    else
-      done?()
+      updateEarnedAchievement 0
 
   else # not alreadyAchieved
     #log.debug 'Creating a new earned achievement called \'' + (achievement.get 'name') + '\' for ' + userID

server/achievements/earned_achievement_handler.coffee

@@ -10,7 +10,7 @@ util = require '../../app/core/utils'
 
 class EarnedAchievementHandler extends Handler
   modelClass: EarnedAchievement
-  
+
   editableProperties: ['notified']
 
   # Don't allow POSTs or anything yet
@@ -141,7 +141,12 @@ class EarnedAchievementHandler extends Handler
       recalculatingAll = true
     t0 = new Date().getTime()
     total = 100000
-    User.count {anonymous:false}, (err, count) -> total = count
+    #testUsers = ['livelily+test37@gmail.com']
+    if testUsers?
+      userQuery = emailLower: {$in: testUsers}
+    else
+      userQuery = anonymous: false
+    User.count userQuery, (err, count) -> total = count
 
     onFinished = ->
       t1 = new Date().getTime()
@@ -159,10 +164,10 @@ class EarnedAchievementHandler extends Handler
     Achievement.find filter, (err, achievements) ->
       callback?(err) if err?
       callback?(new Error 'No achievements to recalculate') unless achievements.length
-      #log.info "Recalculating a total of #{achievements.length} achievements..."
+      log.info "Recalculating a total of #{achievements.length} achievements..."
 
       # Fetch every single user. This tends to get big so do it in a streaming fashion.
-      userStream = User.find().sort('_id').stream()
+      userStream = User.find(userQuery).sort('_id').stream()
       streamFinished = false
       usersTotal = 0
       usersFinished = 0
@@ -253,11 +258,11 @@ class EarnedAchievementHandler extends Handler
               log.error err if err
               #console.log 'User', user.get('name'), 'had newTotalPoints', newTotalPoints, 'and newTotalRewards', newTotalRewards, 'previousRewards', previousRewards
               return doneWithUser(user) unless newTotalPoints or newTotalRewards.gems or _.some(newTotalRewards, (r) -> r.length)
-#              log.debug "Matched a total of #{newTotalPoints} new points"
-#              log.debug "Incrementing score for these achievements with #{newTotalPoints - previousPoints}"
+              #log.debug "Matched a total of #{newTotalPoints} new points"
+              #log.debug "Incrementing score for these achievements with #{newTotalPoints - previousPoints}"
               pointDelta = newTotalPoints - previousPoints
               pctDone = (100 * usersFinished / total).toFixed(2)
-              console.log "Updated points to #{newTotalPoints} (#{if pointDelta < 0 then '' else '+'}#{pointDelta}) for #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
+              #console.log "Updated points to #{newTotalPoints} (#{if pointDelta < 0 then '' else '+'}#{pointDelta}) for #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
               if recalculatingAll
                 update = {$set: {points: newTotalPoints, 'earned.gems': 0, 'earned.heroes': [], 'earned.items': [], 'earned.levels': []}}
               else

server/plugins/achievements.coffee

@@ -33,11 +33,12 @@ AchievablePlugin = (schema, options) ->
       for achievement in loadedAchievements[category]
         do (achievement) ->
           query = achievement.get('query')
-          return log.warn("Empty achievement query for #{achievement.get('name')}.") if _.isEmpty query
+          return log.error("Empty achievement query for #{achievement.get('name')}.") if _.isEmpty query
           isRepeatable = achievement.get('proportionalTo')?
           alreadyAchieved = if isNew then false else LocalMongo.matchesQuery doc.unchangedCopy, query
           newlyAchieved = LocalMongo.matchesQuery(docObj, query)
           return unless newlyAchieved and (not alreadyAchieved or isRepeatable)
+          #log.info "Making an achievement: #{achievement.get('name')} #{achievement.get('_id')} for doc: #{doc.get('name')} #{doc.get('_id')}"
           EarnedAchievement.createForAchievement(achievement, doc, doc.unchangedCopy)
 
 module.exports = AchievablePlugin