diff --git a/config/unicorn.conf.rb b/config/unicorn.conf.rb index 3f97c4742..cd292f3c0 100644 --- a/config/unicorn.conf.rb +++ b/config/unicorn.conf.rb @@ -80,19 +80,39 @@ before_fork do |server, worker| class ::Unicorn::HttpServer alias :master_sleep_orig :master_sleep + def max_rss + `ps -eo rss,args | grep sidekiq | grep -v grep | awk '{print $1}'` + .split("\n") + .map(&:to_i) + .max * 1024 + end + + def out_of_memory? + max_allowed_size = [ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].min; + max_rss > max_allowed_size + end + def check_sidekiq_heartbeat @sidekiq_heartbeat_interval ||= 30.minutes @sidekiq_next_heartbeat_check ||= Time.new.to_i + @sidekiq_heartbeat_interval if @sidekiq_next_heartbeat_check < Time.new.to_i + last_heartbeat = Jobs::RunHeartbeat.last_heartbeat + if out_of_memory? + Rails.logger.warn("Sidekiq is consuming too much memory (using: %0.2fM), restarting" % (max_rss.to_f / 1.megabyte)) + Demon::Sidekiq.restart + end + if last_heartbeat < Time.new.to_i - @sidekiq_heartbeat_interval STDERR.puts "Sidekiq heartbeat test failed, restarting" - puts "Sidekiq heartbeat test failed, restarting" + Rails.logger.warn "Sidekiq heartbeat test failed, restarting" Demon::Sidekiq.restart end @sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval + + $redis.client.disconnect end end