BUGFIX: better resiliency in the backup/restore processes

This commit is contained in:
Régis Hanol 2014-05-13 16:18:08 +02:00
parent 9f4171e487
commit b52177a4b6
3 changed files with 23 additions and 8 deletions

View file

@ -33,10 +33,10 @@ module BackupRestore
end end
def self.mark_as_running! def self.mark_as_running!
# TODO: for more safety, it should acquire a lock # TODO: for extra safety, it should acquire a lock and raise an exception if already running
# and raise an exception if already running!
$redis.set(running_key, "1") $redis.set(running_key, "1")
save_start_logs_message_id save_start_logs_message_id
keep_it_running
end end
def self.is_operation_running? def self.is_operation_running?
@ -80,14 +80,14 @@ module BackupRestore
<<-SQL <<-SQL
DO $$DECLARE row record; DO $$DECLARE row record;
BEGIN BEGIN
-- create "destination" schema if it does not exists already -- create <destination> schema if it does not exists already
-- NOTE: DROP & CREATE SCHEMA is easier, but we don't wont to drop the public schema -- NOTE: DROP & CREATE SCHEMA is easier, but we don't want to drop the public schema
-- ortherwise extensions (like hstore & pg_trgm) won't work anymore -- ortherwise extensions (like hstore & pg_trgm) won't work anymore...
IF NOT EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = '#{destination}') IF NOT EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = '#{destination}')
THEN THEN
CREATE SCHEMA #{destination}; CREATE SCHEMA #{destination};
END IF; END IF;
-- move all "source" tables to "destination" schema -- move all <source> tables to <destination> schema
FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = '#{source}' FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = '#{source}'
LOOP LOOP
EXECUTE 'DROP TABLE IF EXISTS #{destination}.' || quote_ident(row.tablename) || ' CASCADE;'; EXECUTE 'DROP TABLE IF EXISTS #{destination}.' || quote_ident(row.tablename) || ' CASCADE;';
@ -117,6 +117,17 @@ module BackupRestore
"backup_restore_operation_is_running" "backup_restore_operation_is_running"
end end
def self.keep_it_running
# extend the expiry by 1 minute every 30 seconds
Thread.new do
# this thread will be killed when the fork dies
while true
$redis.expire(running_key, 1.minute)
sleep 30.seconds
end
end
end
def self.shutdown_signal_key def self.shutdown_signal_key
"backup_restore_operation_should_shutdown" "backup_restore_operation_should_shutdown"
end end

View file

@ -53,7 +53,7 @@ module Export
@success = true @success = true
"#{@archive_basename}.tar.gz" "#{@archive_basename}.tar.gz"
ensure ensure
notify_user notify_user rescue nil
clean_up clean_up
@success ? log("[SUCCESS]") : log("[FAILED]") @success ? log("[SUCCESS]") : log("[FAILED]")
end end
@ -293,6 +293,8 @@ module Export
def unpause_sidekiq def unpause_sidekiq
log "Unpausing sidekiq..." log "Unpausing sidekiq..."
Sidekiq.unpause! Sidekiq.unpause!
rescue
log "Something went wrong while unpausing Sidekiq."
end end
def disable_readonly_mode def disable_readonly_mode

View file

@ -63,7 +63,7 @@ module Import
else else
@success = true @success = true
ensure ensure
notify_user notify_user rescue nil
clean_up clean_up
@success ? log("[SUCCESS]") : log("[FAILED]") @success ? log("[SUCCESS]") : log("[FAILED]")
end end
@ -312,6 +312,8 @@ module Import
def unpause_sidekiq def unpause_sidekiq
log "Unpausing sidekiq..." log "Unpausing sidekiq..."
Sidekiq.unpause! Sidekiq.unpause!
rescue
log "Something went wrong while unpausing Sidekiq."
end end
def disable_readonly_mode def disable_readonly_mode