Made mail system resilient to redis failures

server/commons/LockManager.coffee

@@ -1,15 +1,26 @@
 config = require '../../server_config'
 redis = require 'redis'
+log = require 'winston'
 
 class LockManager
   constructor: ->
     unless config.isProduction or config.redis.host isnt "localhost"
       throw "You shouldn't be instantiating distributed locks unless in production."
     @redisClient = redis.createClient config.redis.port, config.redis.host
+    @redisClient.on "ready", =>
+      log.info "Redis ready!"
+      @redisNotAvailable = false
+    @redisClient.on "error", (err) =>
+      @redisNotAvailable = true
+      log.error "Redis connection error! Err: #{err}"
+    @redisClient.on "end", =>
+      @redisNotAvailable = true
+      log.error "Redis connection ended!"
     @lockValues = {}
     @unlockScript = "if redis.call(\"get\",KEYS[1]) == ARGV[1] then return redis.call(\"del\",KEYS[1]) else return 0 end"
   
   setLock: (lockName, timeoutMs, cb) =>
+    if @redisNotAvailable is true then return cb "Redis not available!"
     randomNumber = Math.floor(Math.random() * 1000000000)
     @redisClient.set [lockName,randomNumber, "NX", "PX", timeoutMs], (err, res) =>
       if err? then return cb err, null
@@ -20,6 +31,7 @@ class LockManager
         return cb "Lock already set!", null
       
   releaseLock: (lockName, cb) =>
+    if @redisNotAvailable is true then return cb "Redis not available!"
     @redisClient.eval [@unlockScript, 1, lockName, @lockValues[lockName]], (err, res) -> 
       if err? then return cb err, null
       if res

server/routes/mail.coffee

@@ -276,23 +276,25 @@ employerNewCandidatesAvailableTask = ->
 ### End Employer New Candidates Available Email ###
 
 ### New Recruit Leaderboard Email ###
+###
 newRecruitLeaderboardEmailTask = ->
   # tem_kMQFCKX3v4DNAQDsMAsPJC
   #maxRank and maxRankTime should be recorded if isSimulating is false
   mailTaskName = "newRecruitLeaderboardEmailTask"
   lockDurationMs = 6000
   lockManager.setLock mailTaskName, lockDurationMs, (err, lockResult) ->
-
+###
 ### End New Recruit Leaderboard Email ###
   
 ### Employer Matching Candidate Notification Email ### 
+###
 employerMatchingCandidateNotificationTask = ->
   # tem_mYsepTfWQ265noKfZJcbBH
   #save email filters in their own collection
   mailTaskName = "employerMatchingCandidateNotificationTask"
   lockDurationMs = 6000
   lockManager.setLock mailTaskName, lockDurationMs, (err, lockResult) ->
-
+###
 ### End Employer Matching Candidate Notification Email ###
 ### Ladder Update Email ###