2014-02-12 20:32:58 -08:00
module Export
class Exporter
def initialize ( user_id , publish_to_message_bus = false )
@user_id , @publish_to_message_bus = user_id , publish_to_message_bus
ensure_no_operation_is_running
ensure_we_have_a_user
initialize_state
end
def run
log " [STARTED] "
log " ' #{ @user . username } ' has started the backup! "
mark_export_as_running
listen_for_shutdown_signal
2014-03-14 15:53:58 +01:00
ensure_directory_exists ( @tmp_directory )
ensure_directory_exists ( @archive_directory )
write_metadata
### READ-ONLY / START ###
2014-02-12 20:32:58 -08:00
enable_readonly_mode
pause_sidekiq
wait_for_sidekiq
dump_public_schema
2014-03-14 15:53:58 +01:00
disable_readonly_mode
### READ-ONLY / END ###
2014-02-12 20:32:58 -08:00
log " Finalizing backup... "
2014-03-14 15:53:58 +01:00
update_dump
2014-02-12 20:32:58 -08:00
create_archive
2014-03-12 16:23:47 -04:00
after_create_hook
2014-03-11 17:28:12 -04:00
remove_old
2014-02-12 20:32:58 -08:00
rescue SystemExit
log " Backup process was cancelled! "
rescue Exception = > ex
log " EXCEPTION: " + ex . message
log ex . backtrace . join ( " \n " )
else
@success = true
" #{ @archive_basename } .tar.gz "
ensure
2014-03-24 19:34:16 +01:00
notify_user
2014-02-12 20:32:58 -08:00
clean_up
@success ? log ( " [SUCCESS] " ) : log ( " [FAILED] " )
end
protected
def ensure_no_operation_is_running
raise BackupRestore :: OperationRunningError if BackupRestore . is_operation_running?
end
def ensure_we_have_a_user
2014-05-06 14:41:59 +01:00
@user = User . find_by ( id : @user_id )
2014-02-12 20:32:58 -08:00
raise Discourse :: InvalidParameters . new ( :user_id ) unless @user
end
def initialize_state
@success = false
@current_db = RailsMultisite :: ConnectionManagement . current_db
@timestamp = Time . now . strftime ( " %Y-%m-%d-%H%M%S " )
@tmp_directory = File . join ( Rails . root , " tmp " , " backups " , @current_db , @timestamp )
@dump_filename = File . join ( @tmp_directory , BackupRestore :: DUMP_FILE )
@meta_filename = File . join ( @tmp_directory , BackupRestore :: METADATA_FILE )
@archive_directory = File . join ( Rails . root , " public " , " backups " , @current_db )
2014-03-06 16:34:31 -08:00
@archive_basename = File . join ( @archive_directory , " #{ SiteSetting . title . parameterize } - #{ @timestamp } " )
2014-03-24 19:34:16 +01:00
@logs = [ ]
2014-03-28 12:15:53 +01:00
@readonly_mode_was_enabled = Discourse . readonly_mode?
2014-02-12 20:32:58 -08:00
end
def listen_for_shutdown_signal
Thread . new do
while BackupRestore . is_operation_running?
exit if BackupRestore . should_shutdown?
sleep 0 . 1
end
end
end
def mark_export_as_running
log " Marking backup as running... "
BackupRestore . mark_as_running!
end
def enable_readonly_mode
2014-03-28 12:15:53 +01:00
return if @readonly_mode_was_enabled
2014-02-12 20:32:58 -08:00
log " Enabling readonly mode... "
Discourse . enable_readonly_mode
end
def pause_sidekiq
log " Pausing sidekiq... "
Sidekiq . pause!
end
def wait_for_sidekiq
log " Waiting for sidekiq to finish running jobs... "
2014-03-14 15:49:35 +01:00
iterations = 1
while sidekiq_has_running_jobs?
log " Waiting for sidekiq to finish running jobs... # #{ iterations } "
sleep 5
2014-02-12 20:32:58 -08:00
iterations += 1
2014-03-14 15:49:35 +01:00
raise " Sidekiq did not finish running all the jobs in the allowed time! " if iterations > 6
2014-02-12 20:32:58 -08:00
end
end
2014-03-14 15:49:35 +01:00
def sidekiq_has_running_jobs?
Sidekiq :: Workers . new . each do | process_id , thread_id , worker |
payload = worker . try ( :payload )
return true if payload . try ( :all_sites )
return true if payload . try ( :current_site_id ) == @current_db
end
false
end
2014-02-12 20:32:58 -08:00
def write_metadata
log " Writing metadata to ' #{ @meta_filename } '... "
metadata = {
source : " discourse " ,
version : BackupRestore . current_version
}
File . write ( @meta_filename , metadata . to_json )
end
def dump_public_schema
log " Dumping the public schema of the database... "
logs = Queue . new
pg_dump_running = true
Thread . new do
2014-04-07 19:38:47 +02:00
RailsMultisite :: ConnectionManagement :: establish_connection ( db : @current_db )
2014-02-12 20:32:58 -08:00
while pg_dump_running
message = logs . pop . strip
log ( message ) unless message . blank?
end
end
IO . popen ( " #{ pg_dump_command } 2>&1 " ) do | pipe |
begin
while line = pipe . readline
logs << line
end
rescue EOFError
# finished reading...
ensure
pg_dump_running = false
logs << " "
end
end
raise " pg_dump failed " unless $? . success?
end
2014-03-12 11:45:55 +01:00
def pg_dump_command
2014-02-19 15:25:31 +01:00
db_conf = BackupRestore . database_configuration
2014-02-12 20:32:58 -08:00
2014-02-19 15:43:59 +01:00
password_argument = " PGPASSWORD= #{ db_conf . password } " if db_conf . password . present?
2014-02-20 18:42:17 +01:00
host_argument = " --host= #{ db_conf . host } " if db_conf . host . present?
username_argument = " --username= #{ db_conf . username } " if db_conf . username . present?
[ password_argument , # pass the password to pg_dump (if any)
" pg_dump " , # the pg_dump command
" --schema=public " , # only public schema
" --file=' #{ @dump_filename } ' " , # output to the dump.sql file
" --no-owner " , # do not output commands to set ownership of objects
" --no-privileges " , # prevent dumping of access privileges
" --verbose " , # specifies verbose mode
host_argument , # the hostname to connect to (if any)
username_argument , # the username to connect as (if any)
db_conf . database # the name of the database to dump
2014-02-12 20:32:58 -08:00
] . join ( " " )
end
def update_dump
log " Updating dump for more awesomeness... "
` #{ sed_command } `
end
2014-03-12 11:45:55 +01:00
def sed_command
2014-02-12 20:32:58 -08:00
# in order to limit the downtime when restoring as much as possible
# we force the restoration to happen in the "restore" schema
# during the restoration, this make sure we
# - drop the "restore" schema if it exists
# - create the "restore" schema
# - prepend the "restore" schema into the search_path
2014-03-21 11:57:33 -04:00
regexp = " SET search_path = public, pg_catalog; "
2014-02-12 20:32:58 -08:00
replacement = [ " DROP SCHEMA IF EXISTS restore CASCADE; " ,
" CREATE SCHEMA restore; " ,
" SET search_path = restore, public, pg_catalog; " ,
2014-03-21 11:57:33 -04:00
] . join ( " " )
2014-02-12 20:32:58 -08:00
# we only want to replace the VERY first occurence of the search_path command
2014-03-21 11:57:33 -04:00
expression = " 1,/^ #{ regexp } $/s/ #{ regexp } / #{ replacement } / "
2014-02-12 20:32:58 -08:00
# I tried to use the --in-place argument but it was SLOOOWWWWwwwwww
# so I output the result into another file and rename it back afterwards
2014-03-21 11:57:33 -04:00
[ " sed -e ' #{ expression } ' < #{ @dump_filename } > #{ @dump_filename } .tmp " ,
2014-02-12 20:32:58 -08:00
" && " ,
" mv #{ @dump_filename } .tmp #{ @dump_filename } " ,
] . join ( " " )
end
def create_archive
log " Creating archive: #{ File . basename ( @archive_basename ) } .tar.gz "
tar_filename = " #{ @archive_basename } .tar "
log " Making sure archive does not already exist... "
` rm -f #{ tar_filename } `
` rm -f #{ tar_filename } .gz `
log " Creating empty archive... "
` tar --create --file #{ tar_filename } --files-from /dev/null `
log " Archiving metadata... "
FileUtils . cd ( File . dirname ( @meta_filename ) ) do
2014-02-25 19:23:37 +01:00
` tar --append --dereference --file #{ tar_filename } #{ File . basename ( @meta_filename ) } `
2014-02-12 20:32:58 -08:00
end
log " Archiving data dump... "
FileUtils . cd ( File . dirname ( @dump_filename ) ) do
2014-02-25 19:23:37 +01:00
` tar --append --dereference --file #{ tar_filename } #{ File . basename ( @dump_filename ) } `
2014-02-12 20:32:58 -08:00
end
upload_directory = " uploads/ " + @current_db
2014-02-17 19:58:50 +01:00
log " Archiving uploads... "
FileUtils . cd ( File . join ( Rails . root , " public " ) ) do
2014-02-25 19:23:37 +01:00
` tar --append --dereference --file #{ tar_filename } #{ upload_directory } `
2014-02-12 20:32:58 -08:00
end
log " Gzipping archive... "
2014-03-11 12:47:51 +01:00
` gzip --best #{ tar_filename } `
2014-02-12 20:32:58 -08:00
end
2014-03-12 16:23:47 -04:00
def after_create_hook
log " Executing the after_create_hook for the backup "
backup = Backup . create_from_filename ( " #{ File . basename ( @archive_basename ) } .tar.gz " )
backup . after_create_hook
end
2014-03-11 17:28:12 -04:00
def remove_old
log " Removing old backups... "
Backup . remove_old
end
2014-03-24 19:34:16 +01:00
def notify_user
log " Notifying ' #{ @user . username } ' of the end of the backup... "
# NOTE: will only notify if @user != Discourse.site_contact_user
if @success
SystemMessage . create ( @user , :export_succeeded )
else
SystemMessage . create ( @user , :export_failed , logs : @logs . join ( " \n " ) )
end
end
2014-02-12 20:32:58 -08:00
def clean_up
log " Cleaning stuff up... "
remove_tmp_directory
unpause_sidekiq
2014-03-14 15:53:58 +01:00
disable_readonly_mode if Discourse . readonly_mode?
2014-02-12 20:32:58 -08:00
mark_export_as_not_running
log " Finished! "
end
def remove_tmp_directory
log " Removing tmp ' #{ @tmp_directory } ' directory... "
FileUtils . rm_rf ( @tmp_directory ) if Dir [ @tmp_directory ] . present?
rescue
log " Something went wrong while removing the following tmp directory: #{ @tmp_directory } "
end
def unpause_sidekiq
log " Unpausing sidekiq... "
Sidekiq . unpause!
end
def disable_readonly_mode
2014-03-28 12:15:53 +01:00
return if @readonly_mode_was_enabled
2014-02-12 20:32:58 -08:00
log " Disabling readonly mode... "
Discourse . disable_readonly_mode
end
def mark_export_as_not_running
log " Marking backup as finished... "
BackupRestore . mark_as_not_running!
end
def ensure_directory_exists ( directory )
log " Making sure ' #{ directory } ' exists... "
FileUtils . mkdir_p ( directory )
end
def log ( message )
puts ( message ) rescue nil
publish_log ( message ) rescue nil
2014-03-24 19:34:16 +01:00
save_log ( message )
2014-02-12 20:32:58 -08:00
end
def publish_log ( message )
return unless @publish_to_message_bus
data = { timestamp : Time . now , operation : " backup " , message : message }
2014-02-13 10:41:46 -08:00
MessageBus . publish ( BackupRestore :: LOGS_CHANNEL , data , user_ids : [ @user_id ] )
2014-02-12 20:32:58 -08:00
end
2014-03-24 19:34:16 +01:00
def save_log ( message )
@logs << " [ #{ Time . now } ] #{ message } "
end
2014-02-12 20:32:58 -08:00
end
end