diff --git a/server/achievements/earned_achievement_handler.coffee b/server/achievements/earned_achievement_handler.coffee
index 0926bcd28..fadc43dff 100644
--- a/server/achievements/earned_achievement_handler.coffee
+++ b/server/achievements/earned_achievement_handler.coffee
@@ -43,73 +43,82 @@ class EarnedAchievementHandler extends Handler
callback?(new Error 'No achievements to recalculate') unless achievements.length
log.info "Recalculating a total of #{achievements.length} achievements..."
- # Fetch every single user
- User.find {}, (err, users) ->
- callback?(err) if err?
- log.info "for a total of #{users.length} users."
+ # Fetch every single user. This tends to get big so do it in a streaming fashion.
+ userStream = User.find().stream()
+ streamFinished = false
+ usersTotal = 0
+ usersFinished = 0
+ doneWithUser = ->
+ ++usersFinished
+ onFinished?() if streamFinished and usersFinished is usersTotal
+ userStream.on 'error', (err) -> log.error err
+ userStream.on 'close', -> streamFinished = true
+ userStream.on 'data', (user) ->
+ ++usersTotal
- async.each users, ((user, doneWithUser) ->
- # Keep track of a user's already achieved in order to set the notified values correctly
- userID = user.get('_id').toHexString()
+ # Keep track of a user's already achieved in order to set the notified values correctly
+ userID = user.get('_id').toHexString()
- # Fetch all of a user's earned achievements
- EarnedAchievement.find {user: userID}, (err, alreadyEarned) ->
- alreadyEarnedIDs = []
- previousPoints = 0
- async.each alreadyEarned, ((earned, doneWithEarned) ->
- if (_.find achievements, (single) -> earned.get('achievement') is single.get('_id').toHexString()) # if already earned
- alreadyEarnedIDs.push earned.get('achievement')
- previousPoints += earned.get 'earnedPoints'
- doneWithEarned()
- ), -> # After checking already achieved
- # TODO maybe also delete earned? Make sure you don't delete too many
+ # Fetch all of a user's earned achievements
+ EarnedAchievement.find {user: userID}, (err, alreadyEarned) ->
+ alreadyEarnedIDs = []
+ previousPoints = 0
+ async.each alreadyEarned, ((earned, doneWithEarned) ->
+ if (_.find achievements, (single) -> earned.get('achievement') is single.get('_id').toHexString()) # if already earned
+ alreadyEarnedIDs.push earned.get('achievement')
+ previousPoints += earned.get 'earnedPoints'
+ doneWithEarned()
+ ), -> # After checking already achieved
+ # TODO maybe also delete earned? Make sure you don't delete too many
- newTotalPoints = 0
+ newTotalPoints = 0
- async.each achievements, ((achievement, doneWithAchievement) ->
- return doneWithAchievement() unless achievement.isRecalculable()
+ async.each achievements, ((achievement, doneWithAchievement) ->
+ return doneWithAchievement() unless achievement.isRecalculable()
- isRepeatable = achievement.get('proportionalTo')?
- model = mongoose.modelNameByCollection(achievement.get('collection'))
- return doneWithAchievement new Error "Model with collection '#{achievement.get 'collection'}' doesn't exist." unless model?
+ isRepeatable = achievement.get('proportionalTo')?
+ model = mongoose.modelNameByCollection(achievement.get('collection'))
+ return doneWithAchievement new Error "Model with collection '#{achievement.get 'collection'}' doesn't exist." unless model?
- finalQuery = _.clone achievement.get 'query'
- finalQuery.$or = [{}, {}] # Allow both ObjectIDs or hex string IDs
- finalQuery.$or[0][achievement.userField] = userID
- finalQuery.$or[1][achievement.userField] = mongoose.Types.ObjectId userID
+ finalQuery = _.clone achievement.get 'query'
+ finalQuery.$or = [{}, {}] # Allow both ObjectIDs or hex string IDs
+ finalQuery.$or[0][achievement.userField] = userID
+ finalQuery.$or[1][achievement.userField] = mongoose.Types.ObjectId userID
- model.findOne finalQuery, (err, something) ->
- return doneWithAchievement() if _.isEmpty something
+ model.findOne finalQuery, (err, something) ->
+ return doneWithAchievement() if _.isEmpty something
- #log.debug "Matched an achievement: #{achievement.get 'name'} for #{user.get 'name'}"
+ #log.debug "Matched an achievement: #{achievement.get 'name'} for #{user.get 'name'}"
- earned =
- user: userID
- achievement: achievement._id.toHexString()
- achievementName: achievement.get 'name'
- notified: achievement._id in alreadyEarnedIDs
+ earned =
+ user: userID
+ achievement: achievement._id.toHexString()
+ achievementName: achievement.get 'name'
+ notified: achievement._id in alreadyEarnedIDs
- if isRepeatable
- earned.achievedAmount = something.get(achievement.get 'proportionalTo')
- earned.previouslyAchievedAmount = 0
+ if isRepeatable
+ earned.achievedAmount = something.get(achievement.get 'proportionalTo')
+ earned.previouslyAchievedAmount = 0
- expFunction = achievement.getExpFunction()
- newPoints = expFunction(earned.achievedAmount) * achievement.get('worth')
- else
- newPoints = achievement.get 'worth'
+ expFunction = achievement.getExpFunction()
+ newPoints = expFunction(earned.achievedAmount) * achievement.get('worth')
+ else
+ newPoints = achievement.get 'worth'
- earned.earnedPoints = newPoints
- newTotalPoints += newPoints
+ earned.earnedPoints = newPoints
+ newTotalPoints += newPoints
+
+ EarnedAchievement.update {achievement:earned.achievement, user:earned.user}, earned, {upsert: true}, (err) ->
+ doneWithAchievement err
+ ), -> # Wrap up a user, save points
+ # Since some achievements cannot be recalculated it's important to deduct the old amount of exp
+ # and add the new amount, instead of just setting to the new amount
+ return doneWithUser() unless newTotalPoints
+ log.debug "Matched a total of #{newTotalPoints} new points"
+ log.debug "Incrementing score for these achievements with #{newTotalPoints - previousPoints}"
+ User.update {_id: userID}, {$inc: points: newTotalPoints - previousPoints}, {}, (err) ->
+ log.error err if err?
+ doneWithUser()
- EarnedAchievement.update {achievement:earned.achievement, user:earned.user}, earned, {upsert: true}, (err) ->
- doneWithAchievement err
- ), -> # Wrap up a user, save points
- # Since some achievements cannot be recalculated it's important to deduct the old amount of exp
- # and add the new amount, instead of just setting to the new amount
- return doneWithUser() unless newTotalPoints
- log.debug "Matched a total of #{newTotalPoints} new points"
- log.debug "Incrementing score for these achievements with #{newTotalPoints - previousPoints}"
- User.update {_id: userID}, {$inc: points: newTotalPoints - previousPoints}, {}, doneWithUser
- ), onFinished
module.exports = new EarnedAchievementHandler()
diff --git a/server/users/user_handler.coffee b/server/users/user_handler.coffee
index 3fe79fde9..85886726d 100644
--- a/server/users/user_handler.coffee
+++ b/server/users/user_handler.coffee
@@ -208,7 +208,7 @@ UserHandler = class UserHandler extends Handler
@sendSuccess(res, {result: 'success'})
avatar: (req, res, id) ->
- @modelClass.findBySlugOrId(id).exec (err, document) =>
+ @modelClass.findById(id).exec (err, document) =>
return @sendDatabaseError(res, err) if err
return @sendNotFoundError(res) unless document
photoURL = document?.get('photoURL')
@@ -232,7 +232,7 @@ UserHandler = class UserHandler extends Handler
IDify: (idOrSlug, done) ->
return done null, idOrSlug if Handler.isID idOrSlug
- User.findBySlug idOrSlug, (err, user) -> done err, user?.get '_id'
+ User.getBySlug idOrSlug, (err, user) -> done err, user?.get '_id'
getLevelSessions: (req, res, userIDOrSlug) ->
@IDify userIDOrSlug, (err, userID) =>
@@ -391,21 +391,17 @@ UserHandler = class UserHandler extends Handler
countEdits = (model, done) ->
statKey = User.statsMapping.edits[model.modelName]
return done(new Error 'Could not resolve statKey for model') unless statKey?
-
- total = 100000
- User.count {anonymous:false}, (err, count) -> total = count
-
- stream = User.find({anonymous:false}).sort('_id').limit(10).stream()
- numberRunning = 0
- numberRan = 0
- streamClosed = false
- t0 = new Date().getTime()
-
- stream.on 'close', -> streamClosed = true
-
- stream.on 'data', (user) ->
- numberRunning += 1
- stream.pause() if numberRunning > 20
+ userStream = User.find().stream()
+ streamFinished = false
+ usersTotal = 0
+ usersFinished = 0
+ doneWithUser = (err) ->
+ log.error err if err?
+ ++usersFinished
+ done?() if streamFinished and usersFinished is usersTotal
+ userStream.on 'error', (err) -> log.error err
+ userStream.on 'close', -> streamFinished = true
+ userStream.on 'data', (user) ->
userObjectID = user.get('_id')
userStringID = userObjectID.toHexString()
@@ -418,18 +414,7 @@ UserHandler = class UserHandler extends Handler
update.$unset[statKey] = ''
User.findByIdAndUpdate user.get('_id'), update, (err) ->
log.error err if err?
- numberRan += 1
- pctDone = (100 * numberRan / total).toFixed(2)
- console.log "Counted #{statKey} edits for user #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
- numberRunning -= 1
-
- if streamClosed and not numberRunning
- t1 = new Date().getTime()
- runningTime = ((t1-t0)/1000/60/60).toFixed(2)
- console.log "we finished in #{runningTime} hours"
- return done()
-
- stream.resume()
+ doneWithUser()
# I don't like leaking big variables, could remove this for readability
# Meant for passing into MongoDB
@@ -454,62 +439,45 @@ UserHandler = class UserHandler extends Handler
update[method][statName] = count or ''
User.findByIdAndUpdate user.get('_id'), update, doneUpdatingUser
- total = 100000
- User.count {anonymous:false}, (err, count) -> total = count
-
- userStream = User.find({anonymous:false}).sort('_id').stream()
- numberRunning = 0
- numberRan = 0
- streamClosed = false
- t0 = new Date().getTime()
-
- userStream.on 'close', -> streamClosed = true
-
- userStream.on 'data', (user) ->
- numberRunning += 1
- userStream.pause() if numberRunning > 8
+ userStream = User.find().stream()
+ streamFinished = false
+ usersTotal = 0
+ usersFinished = 0
+ doneWithUser = (err) ->
+ log.error err if err?
+ ++usersFinished
+ done?() if streamFinished and usersFinished is usersTotal
+ userStream.on 'error', (err) -> log.error err
+ userStream.on 'close', -> streamFinished = true
+ userStream.on 'data', (user) ->
userObjectID = user.get '_id'
userStringID = userObjectID.toHexString()
# Extend query with a patch ownership test
- _.extend query, {creator: userObjectID}
+ _.extend query, {$or: [{creator: userObjectID}, {creator: userStringID}]}
count = 0
- patchStream = Patch.where(query).stream()
- patchStream.on 'data', (doc) -> ++count if filter doc
-# stream.on 'error', (err) ->
-# updateUser user, count, doneWithUser
-# log.error "Recalculating #{statName} for user #{user} stopped prematurely because of error"
- patchStream.on 'close', ->
- updateUser user, count, ->
- numberRan += 1
- numberRunning -= 1
- pctDone = (100 * numberRan / total).toFixed(2)
- console.log "Counted #{count} #{statName} for user #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
- if streamClosed and not numberRunning
- t1 = new Date().getTime()
- runningTime = ((t1-t0)/1000/60/60).toFixed(2)
- console.log "we finished in #{runningTime} hours"
- return done()
- userStream.resume()
-
+ stream = Patch.where(query).stream()
+ stream.on 'data', (doc) -> ++count if filter doc
+ stream.on 'error', (err) ->
+ updateUser user, count, doneWithUser
+ log.error "Recalculating #{statName} for user #{user} stopped prematurely because of error"
+ stream.on 'close', ->
+ updateUser user, count, doneWithUser
countPatchesByUsers = (query, statName, done) ->
Patch = require '../patches/Patch'
- total = 100000
- User.count {anonymous:false}, (err, count) -> total = count
-
- stream = User.find({anonymous:false}).sort('_id').stream()
- numberRunning = 0
- numberRan = 0
- streamClosed = false
- t0 = new Date().getTime()
-
- stream.on 'close', -> streamClosed = true
-
- stream.on 'data', (user) ->
- numberRunning += 1
- stream.pause() if numberRunning > 50
+ userStream = User.find().stream()
+ streamFinished = false
+ usersTotal = 0
+ usersFinished = 0
+ doneWithUser = (err) ->
+ log.error err if err?
+ ++usersFinished
+ done?() if streamFinished and usersFinished is usersTotal
+ userStream.on 'error', (err) -> log.error err
+ userStream.on 'close', -> streamFinished = true
+ userStream.on 'data', (user) ->
userObjectID = user.get '_id'
userStringID = userObjectID.toHexString()
# Extend query with a patch ownership test
@@ -520,53 +488,28 @@ UserHandler = class UserHandler extends Handler
update = {}
update[method] = {}
update[method][statName] = count or ''
- User.findByIdAndUpdate user.get('_id'), update, ->
- numberRan += 1
- numberRunning -= 1
- pctDone = (100 * numberRan / total).toFixed(2)
- console.log "Counted #{statName} patches for user #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
- if streamClosed and not numberRunning
- t1 = new Date().getTime()
- runningTime = ((t1-t0)/1000/60/60).toFixed(2)
- console.log "we finished in #{runningTime} hours"
- return done()
- stream.resume()
-
+ User.findByIdAndUpdate user.get('_id'), update, doneWithUser
statRecalculators:
gamesCompleted: (done) ->
LevelSession = require '../levels/sessions/LevelSession'
- total = 1000000
- User.count {}, (err, count) -> total = count
-
- stream = User.find().sort('_id').stream()
- numberRunning = 0
- numberRan = 0
- streamClosed = false
- t0 = new Date().getTime()
-
- stream.on 'close', -> streamClosed = true
-
- stream.on 'data', (user) ->
- numberRunning += 1
- stream.pause() if numberRunning > 100
+ userStream = User.find().stream()
+ streamFinished = false
+ usersTotal = 0
+ usersFinished = 0
+ doneWithUser = (err) ->
+ log.error err if err?
+ ++usersFinished
+ done?() if streamFinished and usersFinished is usersTotal
+ userStream.on 'error', (err) -> log.error err
+ userStream.on 'close', -> streamFinished = true
+ userStream.on 'data', (user) ->
userID = user.get('_id').toHexString()
- LevelSession.count {creator: userID, 'state.complete': true}, (err, count) ->
+ LevelSession.count {creator: userID, 'state.completed': true}, (err, count) ->
update = if count then {$set: 'stats.gamesCompleted': count} else {$unset: 'stats.gamesCompleted': ''}
- User.findByIdAndUpdate user.get('_id'), update, ->
- numberRan += 1
- numberRunning -= 1
- pctDone = (100 * numberRan / total).toFixed(2)
- console.log "Counted #{count} levels played for user #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
- if streamClosed and not numberRunning
- t1 = new Date().getTime()
- runningTime = ((t1-t0)/1000/60/60).toFixed(2)
- console.log "we finished in #{runningTime} hours"
- return done()
- stream.resume()
-
+ User.findByIdAndUpdate user.get('_id'), update, doneWithUser
articleEdits: (done) ->
Article = require '../articles/Article'