2014-02-12 20:32:58 -08:00
module Import
class ImportDisabledError < RuntimeError ; end
class FilenameMissingError < RuntimeError ; end
class Importer
def initialize ( user_id , filename , publish_to_message_bus = false )
@user_id , @filename , @publish_to_message_bus = user_id , filename , publish_to_message_bus
ensure_import_is_enabled
ensure_no_operation_is_running
ensure_we_have_a_user
ensure_we_have_a_filename
initialize_state
end
def run
log " [STARTED] "
log " ' #{ @user_info [ :username ] } ' has started the restore! "
mark_import_as_running
listen_for_shutdown_signal
ensure_directory_exists ( @tmp_directory )
copy_archive_to_tmp_directory
unzip_archive
extract_metadata
validate_metadata
extract_dump
restore_dump
2014-04-08 18:06:53 +02:00
### READ-ONLY / START ###
enable_readonly_mode
pause_sidekiq
wait_for_sidekiq
2014-02-12 20:32:58 -08:00
switch_schema!
2014-02-14 00:27:25 +01:00
# TOFIX: MessageBus is busted...
2014-02-12 20:32:58 -08:00
migrate_database
reconnect_database
2014-04-08 18:06:53 +02:00
reload_site_settings
disable_readonly_mode
### READ-ONLY / END ###
2014-02-12 20:32:58 -08:00
extract_uploads
rescue SystemExit
log " Restore process was cancelled! "
rollback
rescue Exception = > ex
log " EXCEPTION: " + ex . message
log ex . backtrace . join ( " \n " )
rollback
else
@success = true
ensure
2014-03-24 19:34:16 +01:00
notify_user
2014-02-12 20:32:58 -08:00
clean_up
@success ? log ( " [SUCCESS] " ) : log ( " [FAILED] " )
end
protected
def ensure_import_is_enabled
2014-02-13 09:19:04 -08:00
raise Import :: ImportDisabledError unless SiteSetting . allow_restore?
2014-02-12 20:32:58 -08:00
end
def ensure_no_operation_is_running
raise BackupRestore :: OperationRunningError if BackupRestore . is_operation_running?
end
def ensure_we_have_a_user
2014-05-06 14:41:59 +01:00
user = User . find_by ( id : @user_id )
2014-02-12 20:32:58 -08:00
raise Discourse :: InvalidParameters . new ( :user_id ) unless user
# keep some user data around to check them against the newly restored database
@user_info = { id : user . id , username : user . username , email : user . email }
end
def ensure_we_have_a_filename
raise Import :: FilenameMissingError if @filename . nil?
end
def initialize_state
@success = false
@current_db = RailsMultisite :: ConnectionManagement . current_db
@current_version = BackupRestore . current_version
@timestamp = Time . now . strftime ( " %Y-%m-%d-%H%M%S " )
@tmp_directory = File . join ( Rails . root , " tmp " , " restores " , @current_db , @timestamp )
@archive_filename = File . join ( @tmp_directory , @filename )
@tar_filename = @archive_filename [ 0 ... - 3 ]
@meta_filename = File . join ( @tmp_directory , BackupRestore :: METADATA_FILE )
@dump_filename = File . join ( @tmp_directory , BackupRestore :: DUMP_FILE )
2014-03-25 15:15:30 +11:00
@logs = [ ]
2014-03-28 12:15:53 +01:00
@readonly_mode_was_enabled = Discourse . readonly_mode?
2014-02-12 20:32:58 -08:00
end
def listen_for_shutdown_signal
Thread . new do
while BackupRestore . is_operation_running?
exit if BackupRestore . should_shutdown?
sleep 0 . 1
end
end
end
def mark_import_as_running
log " Marking restore as running... "
BackupRestore . mark_as_running!
end
def enable_readonly_mode
2014-03-28 12:15:53 +01:00
return if @readonly_mode_was_enabled
2014-02-12 20:32:58 -08:00
log " Enabling readonly mode... "
Discourse . enable_readonly_mode
end
def pause_sidekiq
log " Pausing sidekiq... "
Sidekiq . pause!
end
def wait_for_sidekiq
log " Waiting for sidekiq to finish running jobs... "
2014-03-14 15:49:35 +01:00
iterations = 1
while sidekiq_has_running_jobs?
log " Waiting for sidekiq to finish running jobs... # #{ iterations } "
2014-02-12 20:32:58 -08:00
sleep 5
iterations += 1
2014-03-14 15:49:35 +01:00
raise " Sidekiq did not finish running all the jobs in the allowed time! " if iterations > 6
2014-02-12 20:32:58 -08:00
end
end
2014-03-14 15:49:35 +01:00
def sidekiq_has_running_jobs?
Sidekiq :: Workers . new . each do | process_id , thread_id , worker |
payload = worker . try ( :payload )
return true if payload . try ( :all_sites )
return true if payload . try ( :current_site_id ) == @current_db
end
false
end
2014-02-12 20:32:58 -08:00
def copy_archive_to_tmp_directory
log " Copying archive to tmp directory... "
source = File . join ( Backup . base_directory , @filename )
` cp #{ source } #{ @archive_filename } `
end
def unzip_archive
log " Unzipping archive... "
FileUtils . cd ( @tmp_directory ) { ` gzip --decompress #{ @archive_filename } ` }
end
def extract_metadata
log " Extracting metadata file... "
FileUtils . cd ( @tmp_directory ) { ` tar --extract --file #{ @tar_filename } #{ BackupRestore :: METADATA_FILE } ` }
@metadata = Oj . load_file ( @meta_filename )
end
def validate_metadata
log " Validating metadata... "
log " Current version: #{ @current_version } "
log " Restored version: #{ @metadata [ " version " ] } "
error = " You're trying to import a more recent version of the schema. You should migrate first! "
raise error if @metadata [ " version " ] > @current_version
end
def extract_dump
log " Extracting dump file... "
FileUtils . cd ( @tmp_directory ) { ` tar --extract --file #{ @tar_filename } #{ BackupRestore :: DUMP_FILE } ` }
end
def restore_dump
log " Restoring dump file... (can be quite long) "
logs = Queue . new
psql_running = true
has_error = false
Thread . new do
2014-04-08 18:06:53 +02:00
RailsMultisite :: ConnectionManagement :: establish_connection ( db : @current_db )
2014-02-12 20:32:58 -08:00
while psql_running
message = logs . pop . strip
has_error || = ( message =~ / ERROR: / )
log ( message ) unless message . blank?
end
end
IO . popen ( " #{ psql_command } 2>&1 " ) do | pipe |
begin
while line = pipe . readline
logs << line
end
rescue EOFError
# finished reading...
ensure
psql_running = false
logs << " "
end
end
# psql does not return a valid exit code when an error happens
raise " psql failed " if has_error
end
2014-03-12 11:45:55 +01:00
def psql_command
2014-02-19 15:25:31 +01:00
db_conf = BackupRestore . database_configuration
2014-02-19 15:43:59 +01:00
password_argument = " PGPASSWORD= #{ db_conf . password } " if db_conf . password . present?
2014-02-20 18:42:17 +01:00
host_argument = " --host= #{ db_conf . host } " if db_conf . host . present?
username_argument = " --username= #{ db_conf . username } " if db_conf . username . present?
2014-02-19 15:25:31 +01:00
2014-02-20 18:42:17 +01:00
[ password_argument , # pass the password to psql (if any)
2014-02-19 15:25:31 +01:00
" psql " , # the psql command
" --dbname=' #{ db_conf . database } ' " , # connect to database *dbname*
" --file=' #{ @dump_filename } ' " , # read the dump
" --single-transaction " , # all or nothing (also runs COPY commands faster)
2014-02-20 18:42:17 +01:00
host_argument , # the hostname to connect to (if any)
username_argument # the username to connect as (if any)
2014-02-12 20:32:58 -08:00
] . join ( " " )
end
def switch_schema!
log " Switching schemas... "
2014-02-19 15:25:31 +01:00
sql = [
" BEGIN; " ,
BackupRestore . move_tables_between_schemas_sql ( " public " , " backup " ) ,
BackupRestore . move_tables_between_schemas_sql ( " restore " , " public " ) ,
" COMMIT; "
] . join ( " \n " )
2014-02-12 20:32:58 -08:00
User . exec_sql ( sql )
end
def migrate_database
log " Migrating the database... "
Discourse :: Application . load_tasks
ENV [ " VERSION " ] = @current_version . to_s
2014-02-21 16:17:00 +01:00
Rake :: Task [ " db:migrate " ] . invoke
2014-02-12 20:32:58 -08:00
end
def reconnect_database
log " Reconnecting to the database... "
2014-04-08 18:06:53 +02:00
RailsMultisite :: ConnectionManagement :: establish_connection ( db : @current_db )
end
def reload_site_settings
log " Reloading site settings... "
SiteSetting . refresh!
2014-02-12 20:32:58 -08:00
end
def extract_uploads
log " Extracting uploads... "
if ` tar --list --file #{ @tar_filename } | grep 'uploads/' ` . present?
FileUtils . cd ( File . join ( Rails . root , " public " ) ) do
` tar --extract --keep-newer-files --file #{ @tar_filename } uploads/ `
end
end
end
def rollback
log " Trying to rollback... "
if BackupRestore . can_rollback?
2014-02-14 00:27:25 +01:00
log " Rolling back... "
2014-02-19 15:25:31 +01:00
BackupRestore . move_tables_between_schemas ( " backup " , " public " )
2014-02-12 20:32:58 -08:00
else
2014-02-14 00:27:25 +01:00
log " There was no need to rollback "
2014-02-12 20:32:58 -08:00
end
end
2014-03-24 19:34:16 +01:00
def notify_user
2014-05-06 14:41:59 +01:00
if user = User . find_by ( email : @user_info [ :email ] )
2014-03-24 19:34:16 +01:00
log " Notifying ' #{ user . username } ' of the end of the restore... "
# NOTE: will only notify if user != Discourse.site_contact_user
if @success
SystemMessage . create ( user , :import_succeeded )
else
SystemMessage . create ( user , :import_failed , logs : @logs . join ( " \n " ) )
end
else
log " Could not send notification to ' #{ @user_info [ :username ] } ' ( #{ @user_info [ :email ] } ), because the user does not exists... "
end
end
2014-02-12 20:32:58 -08:00
def clean_up
log " Cleaning stuff up... "
remove_tmp_directory
unpause_sidekiq
2014-04-08 18:06:53 +02:00
disable_readonly_mode if Discourse . readonly_mode?
2014-02-12 20:32:58 -08:00
mark_import_as_not_running
log " Finished! "
end
def remove_tmp_directory
log " Removing tmp ' #{ @tmp_directory } ' directory... "
FileUtils . rm_rf ( @tmp_directory ) if Dir [ @tmp_directory ] . present?
rescue
log " Something went wrong while removing the following tmp directory: #{ @tmp_directory } "
end
def unpause_sidekiq
log " Unpausing sidekiq... "
Sidekiq . unpause!
end
def disable_readonly_mode
2014-03-28 12:15:53 +01:00
return if @readonly_mode_was_enabled
2014-02-12 20:32:58 -08:00
log " Disabling readonly mode... "
Discourse . disable_readonly_mode
end
def mark_import_as_not_running
log " Marking restore as finished... "
BackupRestore . mark_as_not_running!
end
def ensure_directory_exists ( directory )
log " Making sure #{ directory } exists... "
FileUtils . mkdir_p ( directory )
end
def log ( message )
puts ( message ) rescue nil
publish_log ( message ) rescue nil
2014-03-24 19:34:16 +01:00
save_log ( message )
2014-02-12 20:32:58 -08:00
end
def publish_log ( message )
return unless @publish_to_message_bus
data = { timestamp : Time . now , operation : " restore " , message : message }
2014-02-13 10:41:46 -08:00
MessageBus . publish ( BackupRestore :: LOGS_CHANNEL , data , user_ids : [ @user_id ] )
2014-02-12 20:32:58 -08:00
end
2014-03-24 19:34:16 +01:00
def save_log ( message )
@logs << " [ #{ Time . now } ] #{ message } "
end
2014-02-12 20:32:58 -08:00
end
end