Merge pull request #4341 from tgxworld/pipe_dump_to_gzip

FEATURE: Work with compressed version of `pg_dump` during backup and …

lib/backup_restore/backup_restore.rb

@@ -1,3 +1,4 @@
+require "backup_restore/utils"
 require "backup_restore/backuper"
 require "backup_restore/restorer"
 

lib/backup_restore/backuper.rb

@@ -1,6 +1,7 @@
 module BackupRestore
 
   class Backuper
+    include BackupRestore::Utils
 
     attr_reader :success
 
@@ -42,8 +43,6 @@ module BackupRestore
 
       log "Finalizing backup..."
 
-      update_dump
-
       create_archive
 
       after_create_hook
@@ -84,7 +83,7 @@ module BackupRestore
       @current_db = RailsMultisite::ConnectionManagement.current_db
       @timestamp = Time.now.strftime("%Y-%m-%d-%H%M%S")
       @tmp_directory = File.join(Rails.root, "tmp", "backups", @current_db, @timestamp)
-      @dump_filename = File.join(@tmp_directory, BackupRestore::DUMP_FILE)
+      @dump_filename = "#{File.join(@tmp_directory, BackupRestore::DUMP_FILE)}.gz"
       @meta_filename = File.join(@tmp_directory, BackupRestore::METADATA_FILE)
       @archive_directory = File.join(Rails.root, "public", "backups", @current_db)
       @archive_basename = File.join(@archive_directory, "#{SiteSetting.title.parameterize}-#{@timestamp}")
@@ -192,6 +191,7 @@ module BackupRestore
         "--no-owner",                 # do not output commands to set ownership of objects
         "--no-privileges",            # prevent dumping of access privileges
         "--verbose",                  # specifies verbose mode
+        "--compress=4",               # Compression level of 4
         host_argument,                # the hostname to connect to (if any)
         port_argument,                # the port to connect to (if any)
         username_argument,            # the username to connect as (if any)
@@ -199,59 +199,32 @@ module BackupRestore
       ].join(" ")
     end
 
-    def update_dump
-      log "Updating dump for more awesomeness..."
-
-      `#{sed_command}`
-    end
-
-    def sed_command
-      # in order to limit the downtime when restoring as much as possible
-      # we force the restoration to happen in the "restore" schema
-
-      # during the restoration, this make sure we
-      #  - drop the "restore" schema if it exists
-      #  - create the "restore" schema
-      #  - prepend the "restore" schema into the search_path
-
-      regexp = "SET search_path = public, pg_catalog;"
-
-      replacement = [ "DROP SCHEMA IF EXISTS restore CASCADE;",
-                      "CREATE SCHEMA restore;",
-                      "SET search_path = restore, public, pg_catalog;",
-                    ].join(" ")
-
-      # we only want to replace the VERY first occurence of the search_path command
-      expression = "1,/^#{regexp}$/s/#{regexp}/#{replacement}/"
-
-      # I tried to use the --in-place argument but it was SLOOOWWWWwwwwww
-      # so I output the result into another file and rename it back afterwards
-      [ "sed -e '#{expression}' < #{@dump_filename} > #{@dump_filename}.tmp",
-        "&&",
-        "mv #{@dump_filename}.tmp #{@dump_filename}",
-      ].join(" ")
-    end
-
     def create_archive
       log "Creating archive: #{File.basename(@archive_basename)}.tar.gz"
 
       tar_filename = "#{@archive_basename}.tar"
 
       log "Making sure archive does not already exist..."
-      `rm -f #{tar_filename}`
-      `rm -f #{tar_filename}.gz`
+      execute_command("rm -f #{tar_filename}")
+      execute_command("rm -f #{tar_filename}.gz")
 
       log "Creating empty archive..."
-      `tar --create --file #{tar_filename} --files-from /dev/null`
+      execute_command("tar --create --file #{tar_filename} --files-from /dev/null")
 
       log "Archiving data dump..."
-      FileUtils.cd(File.dirname(@dump_filename)) do
-        `tar --append --dereference --file #{tar_filename} #{File.basename(@dump_filename)}`
+      FileUtils.cd(File.dirname("#{@dump_filename}")) do
+        execute_command(
+          "tar --append --dereference --file #{tar_filename} #{File.basename(@dump_filename)}",
+          "Failed to archive data dump."
+        )
       end
 
       log "Archiving metadata..."
       FileUtils.cd(File.dirname(@meta_filename)) do
-        `tar --append --dereference --file #{tar_filename} #{File.basename(@meta_filename)}`
+        execute_command(
+          "tar --append --dereference --file #{tar_filename} #{File.basename(@meta_filename)}",
+          "Failed to archive metadata."
+        )
       end
 
       if @with_uploads
@@ -259,14 +232,17 @@ module BackupRestore
 
         log "Archiving uploads..."
         FileUtils.cd(File.join(Rails.root, "public")) do
-          `tar --append --dereference --file #{tar_filename} #{upload_directory}`
+          execute_command(
+            "tar --append --dereference --file #{tar_filename} #{upload_directory}",
+            "Failed to archive uploads."
+          )
         end
       end
 
       remove_tmp_directory
 
       log "Gzipping archive, this may take a while..."
-      `gzip -5 #{tar_filename}`
+      execute_command("gzip -5 #{tar_filename}", "Failed to gzip archive.")
     end
 
     def after_create_hook

lib/backup_restore/restorer.rb

@@ -4,6 +4,7 @@ module BackupRestore
   class FilenameMissingError < RuntimeError; end
 
   class Restorer
+    include BackupRestore::Utils
 
     attr_reader :success
 
@@ -67,8 +68,13 @@ module BackupRestore
     else
       @success = true
     ensure
-      notify_user rescue nil
-      clean_up
+      begin
+        notify_user
+        clean_up
+      rescue => ex
+        Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n"))
+      end
+
       @success ? log("[SUCCESS]") : log("[FAILED]")
     end
 
@@ -157,17 +163,26 @@ module BackupRestore
     def copy_archive_to_tmp_directory
       log "Copying archive to tmp directory..."
       source = File.join(Backup.base_directory, @filename)
-      `cp '#{source}' '#{@archive_filename}'`
+      execute_command("cp '#{source}' '#{@archive_filename}'", "Failed to copy archive to tmp directory.")
     end
 
     def unzip_archive
       log "Unzipping archive, this may take a while..."
-      FileUtils.cd(@tmp_directory) { `gzip --decompress '#{@archive_filename}'` }
+      FileUtils.cd(@tmp_directory) do
+        execute_command("gzip --decompress '#{@archive_filename}'", "Failed to unzip archive.")
+      end
     end
 
     def extract_metadata
       log "Extracting metadata file..."
-      FileUtils.cd(@tmp_directory) { `tar --extract --file '#{@tar_filename}' #{BackupRestore::METADATA_FILE}` }
+
+      FileUtils.cd(@tmp_directory) do
+        execute_command(
+          "tar --extract --file '#{@tar_filename}' #{BackupRestore::METADATA_FILE}",
+          "Failed to extract metadata file."
+        )
+      end
+
       @metadata = Oj.load_file(@meta_filename)
     end
 
@@ -182,7 +197,13 @@ module BackupRestore
 
     def extract_dump
       log "Extracting dump file..."
-      FileUtils.cd(@tmp_directory) { `tar --extract --file '#{@tar_filename}' #{BackupRestore::DUMP_FILE}` }
+
+      FileUtils.cd(@tmp_directory) do
+        execute_command(
+          "tar --extract --file '#{@tar_filename}' #{BackupRestore::DUMP_FILE}.gz",
+          "Failed to extract dump file."
+        )
+      end
     end
 
     def restore_dump
@@ -201,7 +222,7 @@ module BackupRestore
         end
       end
 
-      IO.popen("#{psql_command} 2>&1") do |pipe|
+      IO.popen("gzip -d < #{@dump_filename}.gz | #{sed_command} | #{psql_command} 2>&1") do |pipe|
         begin
           while line = pipe.readline
             logs << line
@@ -229,7 +250,6 @@ module BackupRestore
       [ password_argument,                # pass the password to psql (if any)
         "psql",                           # the psql command
         "--dbname='#{db_conf.database}'", # connect to database *dbname*
-        "--file='#{@dump_filename}'",     # read the dump
         "--single-transaction",           # all or nothing (also runs COPY commands faster)
         host_argument,                    # the hostname to connect to (if any)
         port_argument,                # the port to connect to (if any)
@@ -237,6 +257,28 @@ module BackupRestore
       ].join(" ")
     end
 
+    def sed_command
+      # in order to limit the downtime when restoring as much as possible
+      # we force the restoration to happen in the "restore" schema
+
+      # during the restoration, this make sure we
+      #  - drop the "restore" schema if it exists
+      #  - create the "restore" schema
+      #  - prepend the "restore" schema into the search_path
+
+      regexp = "SET search_path = public, pg_catalog;"
+
+      replacement = [ "DROP SCHEMA IF EXISTS restore CASCADE;",
+                      "CREATE SCHEMA restore;",
+                      "SET search_path = restore, public, pg_catalog;",
+                    ].join(" ")
+
+      # we only want to replace the VERY first occurence of the search_path command
+      expression = "1,/^#{regexp}$/s/#{regexp}/#{replacement}/"
+
+      "sed -e '#{expression}'"
+    end
+
     def switch_schema!
       log "Switching schemas... try reloading the site in 5 minutes, if successful, then reboot and restore is complete."
 
@@ -279,7 +321,10 @@ module BackupRestore
       if `tar --list --file '#{@tar_filename}' | grep 'uploads/'`.present?
         log "Extracting uploads..."
         FileUtils.cd(File.join(Rails.root, "public")) do
-          `tar --extract --keep-newer-files --file '#{@tar_filename}' uploads/`
+          execute_command(
+            "tar --extract --keep-newer-files --file '#{@tar_filename}' uploads/",
+            "Failed to extract uploads."
+          )
         end
       end
     end

lib/backup_restore/utils.rb

@@ -0,0 +1,14 @@
+module BackupRestore
+  module Utils
+    def execute_command(command, failure_message = "")
+      output = `#{command} 2>&1`
+
+      if !$?.success?
+        failure_message = "#{failure_message}\n" if !failure_message.blank?
+        raise "#{failure_message}#{output}"
+      end
+
+      output
+    end
+  end
+end