diff --git a/lib/import_export/category_exporter.rb b/lib/import_export/category_exporter.rb new file mode 100644 index 000000000..b314c713a --- /dev/null +++ b/lib/import_export/category_exporter.rb @@ -0,0 +1,59 @@ +module ImportExport + class CategoryExporter + + attr_reader :export_data + + def initialize(category_id) + @category = Category.find(category_id) + @subcategories = Category.where(parent_category_id: category_id) + @export_data = { + users: [], + category: nil, + subcategories: [], + topics: [] + } + end + + def perform + puts "Exporting category #{@category.name}...", "" + export_categories + export_topics_and_users + self + end + + + CATEGORY_ATTRS = [:id, :name, :color, :created_at, :user_id, :slug, :description, :text_color, + :auto_close_hours, :logo_url, :background_url, :auto_close_based_on_last_post, + :topic_template, :suppress_from_homepage] + + def export_categories + # description + @export_data[:category] = CATEGORY_ATTRS.inject({}) { |h,a| h[a] = @category.send(a); h } + @subcategories.find_each do |subcat| + @export_data[:subcategories] << CATEGORY_ATTRS.inject({}) { |h,a| h[a] = subcat.send(a); h } + end + self + end + + def export_topics_and_users + all_category_ids = [@category.id] + @subcategories.pluck(:id) + description_topic_ids = Category.where(id: all_category_ids).pluck(:topic_id) + topic_exporter = ImportExport::TopicExporter.new(Topic.where(category_id: all_category_ids).pluck(:id) - description_topic_ids) + topic_exporter.perform + @export_data[:users] = topic_exporter.export_data[:users] + @export_data[:topics] = topic_exporter.export_data[:topics] + self + end + + def save_to_file(filename=nil) + require 'json' + output_basename = filename || File.join("category-export-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json") + File.open(output_basename, "w:UTF-8") do |f| + f.write(@export_data.to_json) + end + puts "Export saved to #{output_basename}" + output_basename + end + + end +end diff --git a/lib/import_export/category_importer.rb b/lib/import_export/category_importer.rb new file mode 100644 index 000000000..b130fe405 --- /dev/null +++ b/lib/import_export/category_importer.rb @@ -0,0 +1,55 @@ +require File.join(Rails.root, 'script', 'import_scripts', 'base.rb') + +module ImportExport + class CategoryImporter < ImportScripts::Base + def initialize(export_data) + @export_data = export_data + @topic_importer = TopicImporter.new(@export_data) + end + + def perform + RateLimiter.disable + + import_users + import_categories + import_topics + self + ensure + RateLimiter.enable + end + + def import_users + @topic_importer.import_users + end + + def import_categories + id = @export_data[:category].delete(:id) + parent = Category.new(@export_data[:category]) + parent.user_id = @topic_importer.new_user_id(@export_data[:category][:user_id]) # imported user's new id + parent.custom_fields["import_id"] = id + parent.save! + set_category_description(parent, @export_data[:category][:description]) + + @export_data[:subcategories].each do |cat_attrs| + id = cat_attrs.delete(:id) + subcategory = Category.new(cat_attrs) + subcategory.parent_category_id = parent.id + subcategory.user_id = @topic_importer.new_user_id(cat_attrs[:user_id]) + subcategory.custom_fields["import_id"] = id + subcategory.save! + set_category_description(subcategory, cat_attrs[:description]) + end + end + + def set_category_description(c, description) + post = c.topic.ordered_posts.first + post.raw = description + post.save! + post.rebake! + end + + def import_topics + @topic_importer.import_topics + end + end +end diff --git a/lib/import_export/import_export.rb b/lib/import_export/import_export.rb new file mode 100644 index 000000000..715a4bea5 --- /dev/null +++ b/lib/import_export/import_export.rb @@ -0,0 +1,26 @@ +require "import_export/category_exporter" +require "import_export/category_importer" +require "import_export/topic_exporter" +require "import_export/topic_importer" +require "json" + +module ImportExport + + def self.export_category(category_id) + ImportExport::CategoryExporter.new(category_id).perform.save_to_file + end + + def self.import_category(filename) + export_data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) }) + ImportExport::CategoryImporter.new(export_data).perform + end + + def self.export_topics(topic_ids) + ImportExport::TopicExporter.new(topic_ids).perform.save_to_file + end + + def self.import_topics(filename) + export_data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) }) + ImportExport::TopicImporter.new(export_data).perform + end +end diff --git a/lib/import_export/topic_exporter.rb b/lib/import_export/topic_exporter.rb new file mode 100644 index 000000000..962416c24 --- /dev/null +++ b/lib/import_export/topic_exporter.rb @@ -0,0 +1,94 @@ +module ImportExport + class TopicExporter + + attr_reader :exported_user_ids, :export_data + + def initialize(topic_ids) + @topic_ids = topic_ids + @exported_user_ids = [] + @export_data = { + users: [], + topics: [] + } + end + + def perform + export_users + export_topics + # TODO: user actions + + self + end + + + USER_ATTRS = [:id, :email, :username, :name, :created_at, :trust_level, :active, :last_emailed_at] + + def export_users + # TODO: avatar + + @exported_user_ids = [] + @topic_ids.each do |topic_id| + t = Topic.find(topic_id) + t.posts.includes(user: [:user_profile]).find_each do |post| + u = post.user + unless @exported_user_ids.include?(u.id) + x = USER_ATTRS.inject({}) { |h, a| h[a] = u.send(a); h; } + @export_data[:users] << x.merge({ + bio_raw: u.user_profile.bio_raw, + website: u.user_profile.website, + location: u.user_profile.location + }) + @exported_user_ids << u.id + end + end + end + + self + end + + + def export_topics + @topic_ids.each do |topic_id| + t = Topic.find(topic_id) + puts t.title + export_topic(t) + end + puts "" + end + + + TOPIC_ATTRS = [:id, :title, :created_at, :views, :category_id, :closed, :archived, :archetype] + POST_ATTRS = [:id, :user_id, :post_number, :raw, :created_at, :reply_to_post_number, + :hidden, :hidden_reason_id, :wiki] + + def export_topic(topic) + topic_data = {} + + TOPIC_ATTRS.each do |a| + topic_data[a] = topic.send(a) + end + + topic_data[:posts] = [] + + topic.ordered_posts.find_each do |post| + topic_data[:posts] << POST_ATTRS.inject({}) { |h, a| h[a] = post.send(a); h; } + end + + @export_data[:topics] << topic_data + + self + end + + + def save_to_file(filename=nil) + require 'json' + output_basename = filename || File.join("topic-export-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json") + File.open(output_basename, "w:UTF-8") do |f| + f.write(@export_data.to_json) + end + puts "Export saved to #{output_basename}" + output_basename + end + + end +end diff --git a/lib/import_export/topic_importer.rb b/lib/import_export/topic_importer.rb new file mode 100644 index 000000000..c101f768a --- /dev/null +++ b/lib/import_export/topic_importer.rb @@ -0,0 +1,67 @@ +require File.join(Rails.root, 'script', 'import_scripts', 'base.rb') + +module ImportExport + class TopicImporter < ImportScripts::Base + def initialize(export_data) + @export_data = export_data + end + + def perform + RateLimiter.disable + + import_users + import_topics + self + ensure + RateLimiter.enable + end + + def import_users + @export_data[:users].each do |u| + existing = User.where(email: u[:email]).first + if existing && existing.custom_fields["import_id"] != u[:id] + existing.custom_fields["import_id"] = u[:id] + existing.save! + else + u = create_user(u, u[:id]) # see ImportScripts::Base + end + end + self + end + + def import_topics + @export_data[:topics].each do |t| + puts "" + print t[:title] + + first_post_attrs = t[:posts].first.merge( t.slice(*(TopicExporter::TOPIC_ATTRS - [:id, :category_id])) ) + first_post_attrs[:user_id] = new_user_id(first_post_attrs[:user_id]) + first_post_attrs[:category] = new_category_id(t[:category_id]) + + first_post = create_post( first_post_attrs, first_post_attrs[:id] ) + topic_id = first_post.topic_id + t[:posts].each_with_index do |post_data, i| + next if i == 0 + print "." + create_post(post_data.merge({ + topic_id: topic_id, + user_id: new_user_id(post_data[:user_id]) + }), post_data[:id]) # see ImportScripts::Base + end + end + + puts "" + + self + end + + def new_user_id(external_user_id) + ucf = UserCustomField.where(name: "import_id", value: external_user_id.to_s).first + ucf ? ucf.user_id : Discourse::SYSTEM_USER_ID + end + + def new_category_id(external_category_id) + CategoryCustomField.where(name: "import_id", value: external_category_id).first.category_id rescue nil + end + end +end diff --git a/script/discourse b/script/discourse index ee847f036..fb91f986f 100755 --- a/script/discourse +++ b/script/discourse @@ -134,12 +134,57 @@ class DiscourseCLI < Thor puts 'Requests sent. Clients will refresh on next navigation.' end + desc "export_category", "Export a category, all its topics, and all users who posted in those topics" + def export_category(category_id) + raise "Category id argument is missing!" unless category_id + + load_rails + load_import_export + ImportExport.export_category(category_id) + puts "", "Done", "" + end + + desc "import_category", "Import a category, its topics and the users from the output of the export_category command" + def import_category(filename) + raise "File name argument missing!" unless filename + + puts "Starting import from #{filename}..." + load_rails + load_import_export + ImportExport.import_category(filename) + puts "", "Done", "" + end + + desc "export_topics", "Export topics and all users who posted in that topic. Accepts multiple topic id's" + def export_topics(*topic_ids) + puts "Starting export of topics...", "" + load_rails + load_import_export + ImportExport.export_topics(topic_ids) + puts "", "Done", "" + end + + desc "import_topics", "Import topics and their users from the output of the export_topic command" + def import_topics(filename) + raise "File name argument missing!" unless filename + + puts "Starting import from #{filename}..." + load_rails + load_import_export + ImportExport.import_topics(filename) + puts "", "Done", "" + end + private def load_rails require File.expand_path(File.dirname(__FILE__) + "/../config/environment") end + def load_import_export + require File.expand_path(File.dirname(__FILE__) + "/../lib/import_export/import_export") + end + def do_remap(from, to) sql = "SELECT table_name, column_name FROM information_schema.columns