require_dependency 'export/json_encoder' require_dependency 'export/export' require_dependency 'import/import' module Jobs class Exporter < Jobs::Base sidekiq_options retry: false def execute(args) raise Import::ImportInProgressError if Import::is_import_running? raise Export::ExportInProgressError if Export::is_export_running? @format = args[:format] || :json @output_base_filename = File.absolute_path( args[:filename] || File.join( Rails.root, 'tmp', "export-#{Time.now.strftime('%Y-%m-%d-%H%M%S')}" ) ) @output_base_filename = @output_base_filename[0...-3] if @output_base_filename[-3..-1] == '.gz' @output_base_filename = @output_base_filename[0...-4] if @output_base_filename[-4..-1] == '.tar' @user = args[:user_id] ? User.where(id: args[:user_id].to_i).first : nil start_export @encoder.write_schema_info( source: 'discourse', version: Export.current_schema_version ) ordered_models_for_export.each do |model| log " #{model.table_name}" column_info = model.columns column_names = model.column_names results = model.connection.raw_connection.async_exec("select * from #{model.table_name}").to_enum @encoder.write_table(model.table_name, column_info) do |num_rows_written| log("#{num_rows_written} rows written") if num_rows_written > 0 rows = [] begin while rows.count < batch_size row = results.next rows << column_names.map{|col| row[col]} end rescue StopIteration # we are done end rows end end "#{@output_base_filename}.tar.gz" ensure finish_export end def ordered_models_for_export Export.models_included_in_export end def order_columns_for(model) @order_columns_for_hash ||= { 'CategoryFeaturedTopic' => 'category_id, topic_id', 'CategorySearchData' => 'category_id', 'PostOneboxRender' => 'post_id, onebox_render_id', 'PostReply' => 'post_id, reply_id', 'PostSearchData' => 'post_id', 'PostTiming' => 'topic_id, post_number, user_id', 'SiteContent' => 'content_type', 'TopicUser' => 'topic_id, user_id', 'UserSearchData' => 'user_id', 'UserStat' => 'user_id', 'View' => 'parent_id, parent_type, ip_address, viewed_at' } @order_columns_for_hash[model.name] end def batch_size 1000 end def start_export if @format == :json @encoder = Export::JsonEncoder.new else raise Export::FormatInvalidError end Export.set_export_started Discourse.enable_maintenance_mode end def finish_export if @encoder @encoder.finish create_tar_file @encoder.remove_tmp_directory('export') end ensure Export.set_export_is_not_running Discourse.disable_maintenance_mode send_notification end def create_tar_file filenames = @encoder.filenames tar_filename = "#{@output_base_filename}.tar" upload_directory = "uploads/" + RailsMultisite::ConnectionManagement.current_db FileUtils.cd(File.join(Rails.root, 'public')) do `tar cvf #{tar_filename} #{upload_directory}` end filenames.each do |filename| FileUtils.cd(File.dirname(filename)) do `tar --append --file=#{tar_filename} #{File.basename(filename)}` end end `gzip #{tar_filename}` true end def send_notification SystemMessage.new(@user).create('export_succeeded') if @user true end end end