codeninjasllc/codecombat
1
0
Fork 0
mirror of https://github.com/codeninjasllc/codecombat.git synced 2025-04-28 23:13:59 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added Query streams wherever possible to ease server load.

Browse source
This commit is contained in:
Ruben Vereecken 2014-08-14 19:17:44 +02:00 • committed by Scott Erickson
parent 34207d28c9
commit 4f22723084
2 changed files with 122 additions and 170 deletions

119
server/achievements/earned_achievement_handler.coffee
View file

@ -43,73 +43,82 @@ class EarnedAchievementHandler extends Handler
callback?(new Error 'No achievements to recalculate') unless achievements.length callback?(new Error 'No achievements to recalculate') unless achievements.length
log.info "Recalculating a total of #{achievements.length} achievements..." log.info "Recalculating a total of #{achievements.length} achievements..."
# Fetch every single user # Fetch every single user. This tends to get big so do it in a streaming fashion.
User.find {}, (err, users) -> userStream = User.find().stream()
callback?(err) if err? streamFinished = false
log.info "for a total of #{users.length} users." usersTotal = 0
usersFinished = 0
doneWithUser = ->
++usersFinished
onFinished?() if streamFinished and usersFinished is usersTotal
userStream.on 'error', (err) -> log.error err
userStream.on 'close', -> streamFinished = true
userStream.on 'data', (user) ->
++usersTotal
async.each users, ((user, doneWithUser) -> # Keep track of a user's already achieved in order to set the notified values correctly
# Keep track of a user's already achieved in order to set the notified values correctly userID = user.get('_id').toHexString()
userID = user.get('_id').toHexString()
# Fetch all of a user's earned achievements # Fetch all of a user's earned achievements
EarnedAchievement.find {user: userID}, (err, alreadyEarned) -> EarnedAchievement.find {user: userID}, (err, alreadyEarned) ->
alreadyEarnedIDs = [] alreadyEarnedIDs = []
previousPoints = 0 previousPoints = 0
async.each alreadyEarned, ((earned, doneWithEarned) -> async.each alreadyEarned, ((earned, doneWithEarned) ->
if (_.find achievements, (single) -> earned.get('achievement') is single.get('_id').toHexString()) # if already earned if (_.find achievements, (single) -> earned.get('achievement') is single.get('_id').toHexString()) # if already earned
alreadyEarnedIDs.push earned.get('achievement') alreadyEarnedIDs.push earned.get('achievement')
previousPoints += earned.get 'earnedPoints' previousPoints += earned.get 'earnedPoints'
doneWithEarned() doneWithEarned()
), -> # After checking already achieved ), -> # After checking already achieved
# TODO maybe also delete earned? Make sure you don't delete too many # TODO maybe also delete earned? Make sure you don't delete too many
newTotalPoints = 0 newTotalPoints = 0
async.each achievements, ((achievement, doneWithAchievement) -> async.each achievements, ((achievement, doneWithAchievement) ->
return doneWithAchievement() unless achievement.isRecalculable() return doneWithAchievement() unless achievement.isRecalculable()
isRepeatable = achievement.get('proportionalTo')? isRepeatable = achievement.get('proportionalTo')?
model = mongoose.modelNameByCollection(achievement.get('collection')) model = mongoose.modelNameByCollection(achievement.get('collection'))
return doneWithAchievement new Error "Model with collection '#{achievement.get 'collection'}' doesn't exist." unless model? return doneWithAchievement new Error "Model with collection '#{achievement.get 'collection'}' doesn't exist." unless model?
finalQuery = _.clone achievement.get 'query' finalQuery = _.clone achievement.get 'query'
finalQuery.$or = [{}, {}] # Allow both ObjectIDs or hex string IDs finalQuery.$or = [{}, {}] # Allow both ObjectIDs or hex string IDs
finalQuery.$or[0][achievement.userField] = userID finalQuery.$or[0][achievement.userField] = userID
finalQuery.$or[1][achievement.userField] = mongoose.Types.ObjectId userID finalQuery.$or[1][achievement.userField] = mongoose.Types.ObjectId userID
model.findOne finalQuery, (err, something) -> model.findOne finalQuery, (err, something) ->
return doneWithAchievement() if _.isEmpty something return doneWithAchievement() if _.isEmpty something
#log.debug "Matched an achievement: #{achievement.get 'name'} for #{user.get 'name'}" #log.debug "Matched an achievement: #{achievement.get 'name'} for #{user.get 'name'}"
earned = earned =
user: userID user: userID
achievement: achievement._id.toHexString() achievement: achievement._id.toHexString()
achievementName: achievement.get 'name' achievementName: achievement.get 'name'
notified: achievement._id in alreadyEarnedIDs notified: achievement._id in alreadyEarnedIDs
if isRepeatable if isRepeatable
earned.achievedAmount = something.get(achievement.get 'proportionalTo') earned.achievedAmount = something.get(achievement.get 'proportionalTo')
earned.previouslyAchievedAmount = 0 earned.previouslyAchievedAmount = 0
expFunction = achievement.getExpFunction() expFunction = achievement.getExpFunction()
newPoints = expFunction(earned.achievedAmount) * achievement.get('worth') newPoints = expFunction(earned.achievedAmount) * achievement.get('worth')
else else
newPoints = achievement.get 'worth' newPoints = achievement.get 'worth'
earned.earnedPoints = newPoints earned.earnedPoints = newPoints
newTotalPoints += newPoints newTotalPoints += newPoints
EarnedAchievement.update {achievement:earned.achievement, user:earned.user}, earned, {upsert: true}, (err) ->
doneWithAchievement err
), -> # Wrap up a user, save points
# Since some achievements cannot be recalculated it's important to deduct the old amount of exp
# and add the new amount, instead of just setting to the new amount
return doneWithUser() unless newTotalPoints
log.debug "Matched a total of #{newTotalPoints} new points"
log.debug "Incrementing score for these achievements with #{newTotalPoints - previousPoints}"
User.update {_id: userID}, {$inc: points: newTotalPoints - previousPoints}, {}, (err) ->
log.error err if err?
doneWithUser()
EarnedAchievement.update {achievement:earned.achievement, user:earned.user}, earned, {upsert: true}, (err) ->
doneWithAchievement err
), -> # Wrap up a user, save points
# Since some achievements cannot be recalculated it's important to deduct the old amount of exp
# and add the new amount, instead of just setting to the new amount
return doneWithUser() unless newTotalPoints
log.debug "Matched a total of #{newTotalPoints} new points"
log.debug "Incrementing score for these achievements with #{newTotalPoints - previousPoints}"
User.update {_id: userID}, {$inc: points: newTotalPoints - previousPoints}, {}, doneWithUser
), onFinished
module.exports = new EarnedAchievementHandler() module.exports = new EarnedAchievementHandler()

173
server/users/user_handler.coffee
View file

@ -208,7 +208,7 @@ UserHandler = class UserHandler extends Handler
@sendSuccess(res, {result: 'success'}) @sendSuccess(res, {result: 'success'})
avatar: (req, res, id) -> avatar: (req, res, id) ->
@modelClass.findBySlugOrId(id).exec (err, document) => @modelClass.findById(id).exec (err, document) =>
return @sendDatabaseError(res, err) if err return @sendDatabaseError(res, err) if err
return @sendNotFoundError(res) unless document return @sendNotFoundError(res) unless document
photoURL = document?.get('photoURL') photoURL = document?.get('photoURL')
@ -232,7 +232,7 @@ UserHandler = class UserHandler extends Handler
IDify: (idOrSlug, done) -> IDify: (idOrSlug, done) ->
return done null, idOrSlug if Handler.isID idOrSlug return done null, idOrSlug if Handler.isID idOrSlug
User.findBySlug idOrSlug, (err, user) -> done err, user?.get '_id' User.getBySlug idOrSlug, (err, user) -> done err, user?.get '_id'
getLevelSessions: (req, res, userIDOrSlug) -> getLevelSessions: (req, res, userIDOrSlug) ->
@IDify userIDOrSlug, (err, userID) => @IDify userIDOrSlug, (err, userID) =>
@ -391,21 +391,17 @@ UserHandler = class UserHandler extends Handler
countEdits = (model, done) -> countEdits = (model, done) ->
statKey = User.statsMapping.edits[model.modelName] statKey = User.statsMapping.edits[model.modelName]
return done(new Error 'Could not resolve statKey for model') unless statKey? return done(new Error 'Could not resolve statKey for model') unless statKey?
userStream = User.find().stream()
total = 100000 streamFinished = false
User.count {anonymous:false}, (err, count) -> total = count usersTotal = 0
usersFinished = 0
stream = User.find({anonymous:false}).sort('_id').limit(10).stream() doneWithUser = (err) ->
numberRunning = 0 log.error err if err?
numberRan = 0 ++usersFinished
streamClosed = false done?() if streamFinished and usersFinished is usersTotal
t0 = new Date().getTime() userStream.on 'error', (err) -> log.error err
userStream.on 'close', -> streamFinished = true
stream.on 'close', -> streamClosed = true userStream.on 'data', (user) ->
stream.on 'data', (user) ->
numberRunning += 1
stream.pause() if numberRunning > 20
userObjectID = user.get('_id') userObjectID = user.get('_id')
userStringID = userObjectID.toHexString() userStringID = userObjectID.toHexString()
@ -418,18 +414,7 @@ UserHandler = class UserHandler extends Handler
update.$unset[statKey] = '' update.$unset[statKey] = ''
User.findByIdAndUpdate user.get('_id'), update, (err) -> User.findByIdAndUpdate user.get('_id'), update, (err) ->
log.error err if err? log.error err if err?
numberRan += 1 doneWithUser()
pctDone = (100 * numberRan / total).toFixed(2)
console.log "Counted #{statKey} edits for user #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
numberRunning -= 1
if streamClosed and not numberRunning
t1 = new Date().getTime()
runningTime = ((t1-t0)/1000/60/60).toFixed(2)
console.log "we finished in #{runningTime} hours"
return done()
stream.resume()
# I don't like leaking big variables, could remove this for readability # I don't like leaking big variables, could remove this for readability
# Meant for passing into MongoDB # Meant for passing into MongoDB
@ -454,62 +439,45 @@ UserHandler = class UserHandler extends Handler
update[method][statName] = count or '' update[method][statName] = count or ''
User.findByIdAndUpdate user.get('_id'), update, doneUpdatingUser User.findByIdAndUpdate user.get('_id'), update, doneUpdatingUser
total = 100000 userStream = User.find().stream()
User.count {anonymous:false}, (err, count) -> total = count streamFinished = false
usersTotal = 0
userStream = User.find({anonymous:false}).sort('_id').stream() usersFinished = 0
numberRunning = 0 doneWithUser = (err) ->
numberRan = 0 log.error err if err?
streamClosed = false ++usersFinished
t0 = new Date().getTime() done?() if streamFinished and usersFinished is usersTotal
userStream.on 'error', (err) -> log.error err
userStream.on 'close', -> streamClosed = true userStream.on 'close', -> streamFinished = true
userStream.on 'data', (user) ->
userStream.on 'data', (user) ->
numberRunning += 1
userStream.pause() if numberRunning > 8
userObjectID = user.get '_id' userObjectID = user.get '_id'
userStringID = userObjectID.toHexString() userStringID = userObjectID.toHexString()
# Extend query with a patch ownership test # Extend query with a patch ownership test
_.extend query, {creator: userObjectID} _.extend query, {$or: [{creator: userObjectID}, {creator: userStringID}]}
count = 0 count = 0
patchStream = Patch.where(query).stream() stream = Patch.where(query).stream()
patchStream.on 'data', (doc) -> ++count if filter doc stream.on 'data', (doc) -> ++count if filter doc
# stream.on 'error', (err) -> stream.on 'error', (err) ->
# updateUser user, count, doneWithUser updateUser user, count, doneWithUser
# log.error "Recalculating #{statName} for user #{user} stopped prematurely because of error" log.error "Recalculating #{statName} for user #{user} stopped prematurely because of error"
patchStream.on 'close', -> stream.on 'close', ->
updateUser user, count, -> updateUser user, count, doneWithUser
numberRan += 1
numberRunning -= 1
pctDone = (100 * numberRan / total).toFixed(2)
console.log "Counted #{count} #{statName} for user #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
if streamClosed and not numberRunning
t1 = new Date().getTime()
runningTime = ((t1-t0)/1000/60/60).toFixed(2)
console.log "we finished in #{runningTime} hours"
return done()
userStream.resume()
countPatchesByUsers = (query, statName, done) -> countPatchesByUsers = (query, statName, done) ->
Patch = require '../patches/Patch' Patch = require '../patches/Patch'
total = 100000 userStream = User.find().stream()
User.count {anonymous:false}, (err, count) -> total = count streamFinished = false
usersTotal = 0
stream = User.find({anonymous:false}).sort('_id').stream() usersFinished = 0
numberRunning = 0 doneWithUser = (err) ->
numberRan = 0 log.error err if err?
streamClosed = false ++usersFinished
t0 = new Date().getTime() done?() if streamFinished and usersFinished is usersTotal
userStream.on 'error', (err) -> log.error err
stream.on 'close', -> streamClosed = true userStream.on 'close', -> streamFinished = true
userStream.on 'data', (user) ->
stream.on 'data', (user) ->
numberRunning += 1
stream.pause() if numberRunning > 50
userObjectID = user.get '_id' userObjectID = user.get '_id'
userStringID = userObjectID.toHexString() userStringID = userObjectID.toHexString()
# Extend query with a patch ownership test # Extend query with a patch ownership test
@ -520,53 +488,28 @@ UserHandler = class UserHandler extends Handler
update = {} update = {}
update[method] = {} update[method] = {}
update[method][statName] = count or '' update[method][statName] = count or ''
User.findByIdAndUpdate user.get('_id'), update, -> User.findByIdAndUpdate user.get('_id'), update, doneWithUser
numberRan += 1
numberRunning -= 1
pctDone = (100 * numberRan / total).toFixed(2)
console.log "Counted #{statName} patches for user #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
if streamClosed and not numberRunning
t1 = new Date().getTime()
runningTime = ((t1-t0)/1000/60/60).toFixed(2)
console.log "we finished in #{runningTime} hours"
return done()
stream.resume()
statRecalculators: statRecalculators:
gamesCompleted: (done) -> gamesCompleted: (done) ->
LevelSession = require '../levels/sessions/LevelSession' LevelSession = require '../levels/sessions/LevelSession'
total = 1000000 userStream = User.find().stream()
User.count {}, (err, count) -> total = count streamFinished = false
usersTotal = 0
stream = User.find().sort('_id').stream() usersFinished = 0
numberRunning = 0 doneWithUser = (err) ->
numberRan = 0 log.error err if err?
streamClosed = false ++usersFinished
t0 = new Date().getTime() done?() if streamFinished and usersFinished is usersTotal
userStream.on 'error', (err) -> log.error err
stream.on 'close', -> streamClosed = true userStream.on 'close', -> streamFinished = true
userStream.on 'data', (user) ->
stream.on 'data', (user) ->
numberRunning += 1
stream.pause() if numberRunning > 100
userID = user.get('_id').toHexString() userID = user.get('_id').toHexString()
LevelSession.count {creator: userID, 'state.complete': true}, (err, count) -> LevelSession.count {creator: userID, 'state.completed': true}, (err, count) ->
update = if count then {$set: 'stats.gamesCompleted': count} else {$unset: 'stats.gamesCompleted': ''} update = if count then {$set: 'stats.gamesCompleted': count} else {$unset: 'stats.gamesCompleted': ''}
User.findByIdAndUpdate user.get('_id'), update, -> User.findByIdAndUpdate user.get('_id'), update, doneWithUser
numberRan += 1
numberRunning -= 1
pctDone = (100 * numberRan / total).toFixed(2)
console.log "Counted #{count} levels played for user #{user.get('name') or '???'} (#{user.get('_id')}) (#{pctDone}%)"
if streamClosed and not numberRunning
t1 = new Date().getTime()
runningTime = ((t1-t0)/1000/60/60).toFixed(2)
console.log "we finished in #{runningTime} hours"
return done()
stream.resume()
articleEdits: (done) -> articleEdits: (done) ->
Article = require '../articles/Article' Article = require '../articles/Article'