FEATURE: export/import topics and categories from one Discourse site to another. (Early-access alpha greenlight. More to do...)

lib/import_export/category_exporter.rb

@@ -0,0 +1,59 @@
+module ImportExport
+  class CategoryExporter
+
+    attr_reader :export_data
+
+    def initialize(category_id)
+      @category = Category.find(category_id)
+      @subcategories = Category.where(parent_category_id: category_id)
+      @export_data = {
+        users: [],
+        category: nil,
+        subcategories: [],
+        topics: []
+      }
+    end
+
+    def perform
+      puts "Exporting category #{@category.name}...", ""
+      export_categories
+      export_topics_and_users
+      self
+    end
+
+
+    CATEGORY_ATTRS = [:id, :name, :color, :created_at, :user_id, :slug, :description, :text_color,
+                      :auto_close_hours, :logo_url, :background_url, :auto_close_based_on_last_post,
+                      :topic_template, :suppress_from_homepage]
+
+    def export_categories
+      # description
+      @export_data[:category] = CATEGORY_ATTRS.inject({}) { |h,a| h[a] = @category.send(a); h }
+      @subcategories.find_each do |subcat|
+        @export_data[:subcategories] << CATEGORY_ATTRS.inject({}) { |h,a| h[a] = subcat.send(a); h }
+      end
+      self
+    end
+
+    def export_topics_and_users
+      all_category_ids = [@category.id] + @subcategories.pluck(:id)
+      description_topic_ids = Category.where(id: all_category_ids).pluck(:topic_id)
+      topic_exporter = ImportExport::TopicExporter.new(Topic.where(category_id: all_category_ids).pluck(:id) - description_topic_ids)
+      topic_exporter.perform
+      @export_data[:users]  = topic_exporter.export_data[:users]
+      @export_data[:topics] = topic_exporter.export_data[:topics]
+      self
+    end
+
+    def save_to_file(filename=nil)
+      require 'json'
+      output_basename = filename || File.join("category-export-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json")
+      File.open(output_basename, "w:UTF-8") do |f|
+        f.write(@export_data.to_json)
+      end
+      puts "Export saved to #{output_basename}"
+      output_basename
+    end
+
+  end
+end

lib/import_export/category_importer.rb

@@ -0,0 +1,55 @@
+require File.join(Rails.root, 'script', 'import_scripts', 'base.rb')
+
+module ImportExport
+  class CategoryImporter < ImportScripts::Base
+    def initialize(export_data)
+      @export_data = export_data
+      @topic_importer = TopicImporter.new(@export_data)
+    end
+
+    def perform
+      RateLimiter.disable
+
+      import_users
+      import_categories
+      import_topics
+      self
+    ensure
+      RateLimiter.enable
+    end
+
+    def import_users
+      @topic_importer.import_users
+    end
+
+    def import_categories
+      id = @export_data[:category].delete(:id)
+      parent = Category.new(@export_data[:category])
+      parent.user_id = @topic_importer.new_user_id(@export_data[:category][:user_id]) # imported user's new id
+      parent.custom_fields["import_id"] = id
+      parent.save!
+      set_category_description(parent, @export_data[:category][:description])
+
+      @export_data[:subcategories].each do |cat_attrs|
+        id = cat_attrs.delete(:id)
+        subcategory = Category.new(cat_attrs)
+        subcategory.parent_category_id = parent.id
+        subcategory.user_id = @topic_importer.new_user_id(cat_attrs[:user_id])
+        subcategory.custom_fields["import_id"] = id
+        subcategory.save!
+        set_category_description(subcategory, cat_attrs[:description])
+      end
+    end
+
+    def set_category_description(c, description)
+      post = c.topic.ordered_posts.first
+      post.raw = description
+      post.save!
+      post.rebake!
+    end
+
+    def import_topics
+      @topic_importer.import_topics
+    end
+  end
+end

lib/import_export/import_export.rb

@@ -0,0 +1,26 @@
+require "import_export/category_exporter"
+require "import_export/category_importer"
+require "import_export/topic_exporter"
+require "import_export/topic_importer"
+require "json"
+
+module ImportExport
+
+  def self.export_category(category_id)
+    ImportExport::CategoryExporter.new(category_id).perform.save_to_file
+  end
+
+  def self.import_category(filename)
+    export_data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) })
+    ImportExport::CategoryImporter.new(export_data).perform
+  end
+
+  def self.export_topics(topic_ids)
+    ImportExport::TopicExporter.new(topic_ids).perform.save_to_file
+  end
+
+  def self.import_topics(filename)
+    export_data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) })
+    ImportExport::TopicImporter.new(export_data).perform
+  end
+end

lib/import_export/topic_exporter.rb

@@ -0,0 +1,94 @@
+module ImportExport
+  class TopicExporter
+
+    attr_reader :exported_user_ids, :export_data
+
+    def initialize(topic_ids)
+      @topic_ids = topic_ids
+      @exported_user_ids = []
+      @export_data = {
+        users: [],
+        topics: []
+      }
+    end
+
+    def perform
+      export_users
+      export_topics
+      # TODO: user actions
+
+      self
+    end
+
+
+    USER_ATTRS = [:id, :email, :username, :name, :created_at, :trust_level, :active, :last_emailed_at]
+
+    def export_users
+      # TODO: avatar
+
+      @exported_user_ids = []
+      @topic_ids.each do |topic_id|
+        t = Topic.find(topic_id)
+        t.posts.includes(user: [:user_profile]).find_each do |post|
+          u = post.user
+          unless @exported_user_ids.include?(u.id)
+            x = USER_ATTRS.inject({}) { |h, a| h[a] = u.send(a); h; }
+            @export_data[:users] << x.merge({
+              bio_raw: u.user_profile.bio_raw,
+              website: u.user_profile.website,
+              location: u.user_profile.location
+            })
+            @exported_user_ids << u.id
+          end
+        end
+      end
+
+      self
+    end
+
+
+    def export_topics
+      @topic_ids.each do |topic_id|
+        t = Topic.find(topic_id)
+        puts t.title
+        export_topic(t)
+      end
+      puts ""
+    end
+
+
+    TOPIC_ATTRS = [:id, :title, :created_at, :views, :category_id, :closed, :archived, :archetype]
+    POST_ATTRS = [:id, :user_id, :post_number, :raw, :created_at, :reply_to_post_number,
+                  :hidden, :hidden_reason_id, :wiki]
+
+    def export_topic(topic)
+      topic_data = {}
+
+      TOPIC_ATTRS.each do |a|
+        topic_data[a] = topic.send(a)
+      end
+
+      topic_data[:posts] = []
+
+      topic.ordered_posts.find_each do |post|
+        topic_data[:posts] << POST_ATTRS.inject({}) { |h, a| h[a] = post.send(a); h; }
+      end
+
+      @export_data[:topics] << topic_data
+
+      self
+    end
+
+
+    def save_to_file(filename=nil)
+      require 'json'
+      output_basename = filename || File.join("topic-export-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json")
+      File.open(output_basename, "w:UTF-8") do |f|
+        f.write(@export_data.to_json)
+      end
+      puts "Export saved to #{output_basename}"
+      output_basename
+    end
+
+  end
+end

lib/import_export/topic_importer.rb

@@ -0,0 +1,67 @@
+require File.join(Rails.root, 'script', 'import_scripts', 'base.rb')
+
+module ImportExport
+  class TopicImporter < ImportScripts::Base
+    def initialize(export_data)
+      @export_data = export_data
+    end
+
+    def perform
+      RateLimiter.disable
+
+      import_users
+      import_topics
+      self
+    ensure
+      RateLimiter.enable
+    end
+
+    def import_users
+      @export_data[:users].each do |u|
+        existing = User.where(email: u[:email]).first
+        if existing && existing.custom_fields["import_id"] != u[:id]
+          existing.custom_fields["import_id"] = u[:id]
+          existing.save!
+        else
+          u = create_user(u, u[:id]) # see ImportScripts::Base
+        end
+      end
+      self
+    end
+
+    def import_topics
+      @export_data[:topics].each do |t|
+        puts ""
+        print t[:title]
+
+        first_post_attrs = t[:posts].first.merge( t.slice(*(TopicExporter::TOPIC_ATTRS - [:id, :category_id])) )
+        first_post_attrs[:user_id] = new_user_id(first_post_attrs[:user_id])
+        first_post_attrs[:category] = new_category_id(t[:category_id])
+
+        first_post = create_post( first_post_attrs, first_post_attrs[:id] )
+        topic_id = first_post.topic_id
+        t[:posts].each_with_index do |post_data, i|
+          next if i == 0
+          print "."
+          create_post(post_data.merge({
+            topic_id: topic_id,
+            user_id: new_user_id(post_data[:user_id])
+          }), post_data[:id]) # see ImportScripts::Base
+        end
+      end
+
+      puts ""
+
+      self
+    end
+
+    def new_user_id(external_user_id)
+      ucf = UserCustomField.where(name: "import_id", value: external_user_id.to_s).first
+      ucf ? ucf.user_id : Discourse::SYSTEM_USER_ID
+    end
+
+    def new_category_id(external_category_id)
+      CategoryCustomField.where(name: "import_id", value: external_category_id).first.category_id rescue nil
+    end
+  end
+end

script/discourse

@@ -134,12 +134,57 @@ class DiscourseCLI < Thor
     puts 'Requests sent. Clients will refresh on next navigation.'
   end
 
+  desc "export_category", "Export a category, all its topics, and all users who posted in those topics"
+  def export_category(category_id)
+    raise "Category id argument is missing!" unless category_id
+
+    load_rails
+    load_import_export
+    ImportExport.export_category(category_id)
+    puts "", "Done", ""
+  end
+
+  desc "import_category", "Import a category, its topics and the users from the output of the export_category command"
+  def import_category(filename)
+    raise "File name argument missing!" unless filename
+
+    puts "Starting import from #{filename}..."
+    load_rails
+    load_import_export
+    ImportExport.import_category(filename)
+    puts "", "Done", ""
+  end
+
+  desc "export_topics", "Export topics and all users who posted in that topic. Accepts multiple topic id's"
+  def export_topics(*topic_ids)
+    puts "Starting export of topics...", ""
+    load_rails
+    load_import_export
+    ImportExport.export_topics(topic_ids)
+    puts "", "Done", ""
+  end
+
+  desc "import_topics", "Import topics and their users from the output of the export_topic command"
+  def import_topics(filename)
+    raise "File name argument missing!" unless filename
+
+    puts "Starting import from #{filename}..."
+    load_rails
+    load_import_export
+    ImportExport.import_topics(filename)
+    puts "", "Done", ""
+  end
+
   private
 
   def load_rails
     require File.expand_path(File.dirname(__FILE__) + "/../config/environment")
   end
 
+  def load_import_export
+    require File.expand_path(File.dirname(__FILE__) + "/../lib/import_export/import_export")
+  end
+
   def do_remap(from, to)
     sql = "SELECT table_name, column_name
 FROM information_schema.columns