From 43eb030402fbf0be6980fa8cc250be038bd4c1aa Mon Sep 17 00:00:00 2001
From: Sam <sam.saffron@gmail.com>
Date: Fri, 27 Mar 2015 14:27:01 +1100
Subject: [PATCH] PERF: restart sidekiq if it consumes more than 500MB
configurable with UNICORN_SIDEKIQ_MAX_RSS
---
config/unicorn.conf.rb | 22 +++++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/config/unicorn.conf.rb b/config/unicorn.conf.rb
index 3f97c4742..cd292f3c0 100644
--- a/config/unicorn.conf.rb
+++ b/config/unicorn.conf.rb
@@ -80,19 +80,39 @@ before_fork do |server, worker|
class ::Unicorn::HttpServer
alias :master_sleep_orig :master_sleep
+ def max_rss
+ `ps -eo rss,args | grep sidekiq | grep -v grep | awk '{print $1}'`
+ .split("\n")
+ .map(&:to_i)
+ .max * 1024
+ end
+
+ def out_of_memory?
+ max_allowed_size = [ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].min;
+ max_rss > max_allowed_size
+ end
+
def check_sidekiq_heartbeat
@sidekiq_heartbeat_interval ||= 30.minutes
@sidekiq_next_heartbeat_check ||= Time.new.to_i + @sidekiq_heartbeat_interval
if @sidekiq_next_heartbeat_check < Time.new.to_i
+
last_heartbeat = Jobs::RunHeartbeat.last_heartbeat
+ if out_of_memory?
+ Rails.logger.warn("Sidekiq is consuming too much memory (using: %0.2fM), restarting" % (max_rss.to_f / 1.megabyte))
+ Demon::Sidekiq.restart
+ end
+
if last_heartbeat < Time.new.to_i - @sidekiq_heartbeat_interval
STDERR.puts "Sidekiq heartbeat test failed, restarting"
- puts "Sidekiq heartbeat test failed, restarting"
+ Rails.logger.warn "Sidekiq heartbeat test failed, restarting"
Demon::Sidekiq.restart
end
@sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval
+
+ $redis.client.disconnect
end
end