FEATURE: smf2 import script

script/import_scripts/base.rb

@@ -19,12 +19,18 @@ class ImportScripts::Base
     require File.expand_path(File.dirname(__FILE__) + "/../../config/environment")
 
     @bbcode_to_md = true if ARGV.include?('bbcode-to-md')
+    @existing_groups = {}
+    @failed_groups = []
     @existing_users = {}
     @failed_users = []
     @categories = {}
     @posts = {}
     @topic_lookup = {}
 
+    GroupCustomField.where(name: 'import_id').pluck(:group_id, :value).each do |group_id, import_id|
+      @existing_groups[import_id] = group_id
+    end
+
     UserCustomField.where(name: 'import_id').pluck(:user_id, :value).each do |user_id, import_id|
       @existing_users[import_id] = user_id
     end
@@ -84,6 +90,15 @@ class ImportScripts::Base
     post_id ? @topic_lookup[post_id] : nil
   end
 
+  # Get the Discourse Group id based on the id of the source group
+  def group_id_from_imported_group_id(import_id)
+    @existing_groups[import_id] || @existing_groups[import_id.to_s] || find_group_by_import_id(import_id).try(:id)
+  end
+
+  def find_group_by_import_id(import_id)
+    GroupCustomField.where(name: 'import_id', value: import_id.to_s).first.try(:group)
+  end
+
   # Get the Discourse User id based on the id of the source user
   def user_id_from_imported_user_id(import_id)
     @existing_users[import_id] || @existing_users[import_id.to_s] || find_user_by_import_id(import_id).try(:id)
@@ -110,6 +125,54 @@ class ImportScripts::Base
     admin
   end
 
+  # Iterate through a list of groups to be imported.
+  # Takes a collection and yields to the block for each element.
+  # Block should return a hash with the attributes for each element.
+  # Required fields are :id and :name, where :id is the id of the
+  # group in the original datasource. The given id will not be used
+  # to create the Discourse group record.
+  def create_groups(results, opts={})
+    groups_created = 0
+    groups_skipped = 0
+    total = opts[:total] || results.size
+
+    results.each do |result|
+      g = yield(result)
+
+      if group_id_from_imported_group_id(g[:id])
+        groups_skipped += 1
+      else
+        new_group = create_group(g, g[:id])
+
+        if new_group.valid?
+          @existing_groups[g[:id].to_s] = new_group.id
+          groups_created += 1
+        else
+          @failed_groups << g
+          puts "Failed to create group id #{g[:id]} #{new_group.name}: #{new_group.errors.full_messages}"
+        end
+      end
+
+      print_status groups_created + groups_skipped + @failed_groups.length + (opts[:offset] || 0), total
+    end
+
+    return [groups_created, groups_skipped]
+  end
+
+  def create_group(opts, import_id)
+    opts = opts.dup.tap {|o| o.delete(:id) }
+    import_name = opts[:name]
+    opts[:name] = UserNameSuggester.suggest(import_name)
+
+    existing = Group.where(name: opts[:name]).first
+    return existing if existing and existing.custom_fields["import_id"].to_i == import_id.to_i
+    g = existing || Group.new(opts)
+    g.custom_fields["import_id"] = import_id
+    g.custom_fields["import_name"] = import_name
+
+    g.tap(&:save)
+  end
+
   # Iterate through a list of user records to be imported.
   # Takes a collection, and yields to the block for each element.
   # Block should return a hash with the attributes for the User model.
@@ -151,6 +214,7 @@ class ImportScripts::Base
 
   def create_user(opts, import_id)
     opts.delete(:id)
+    post_create_action = opts.delete(:post_create_action)
     existing = User.where(email: opts[:email].downcase, username: opts[:username]).first
     return existing if existing and existing.custom_fields["import_id"].to_i == import_id.to_i
 
@@ -181,6 +245,7 @@ class ImportScripts::Base
         u = existing
       end
     end
+    post_create_action.try(:call, u) if u.persisted?
 
     u # If there was an error creating the user, u.errors has the messages
   end
@@ -214,6 +279,7 @@ class ImportScripts::Base
     existing = category_from_imported_category_id(import_id)
     return existing if existing
 
+    post_create_action = opts.delete(:post_create_action)
     new_category = Category.new(
       name: opts[:name],
       user_id: -1,
@@ -223,6 +289,7 @@ class ImportScripts::Base
     )
     new_category.custom_fields["import_id"] = import_id if import_id
     new_category.save!
+    post_create_action.try(:call, new_category)
     new_category
   end
 
@@ -279,6 +346,7 @@ class ImportScripts::Base
 
   def create_post(opts, import_id)
     user = User.find(opts[:user_id])
+    post_create_action = opts.delete(:post_create_action)
     opts = opts.merge(skip_validations: true)
     opts[:import_mode] = true
     opts[:custom_fields] ||= {}
@@ -290,9 +358,25 @@ class ImportScripts::Base
 
     post_creator = PostCreator.new(user, opts)
     post = post_creator.create
+    post_create_action.try(:call, post) if post
     post ? post : post_creator.errors.full_messages
   end
 
+  # Creates an upload.
+  # Expects path to be the full path and filename of the source file.
+  def create_upload(user_id, path, source_filename)
+    tmp = Tempfile.new('discourse-upload')
+    src = File.open(path)
+    FileUtils.copy_stream(src, tmp)
+    src.close
+    tmp.rewind
+
+    Upload.create_for(user_id, tmp, source_filename, File.size(tmp))
+  ensure
+    tmp.close rescue nil
+    tmp.unlink rescue nil
+  end
+
   def close_inactive_topics(opts={})
     num_days = opts[:days] || 30
     puts '', "Closing topics that have been inactive for more than #{num_days} days."

script/import_scripts/smf2.rb

@@ -0,0 +1,527 @@
+require File.expand_path(File.dirname(__FILE__) + '/base.rb')
+
+require 'mysql2'
+require 'color'
+require 'htmlentities'
+require 'tsort'
+require 'set'
+require 'optparse'
+
+class ImportScripts::Smf2 < ImportScripts::Base
+
+  def self.run
+    options = Options.new
+    begin
+      options.parse!
+    rescue Options::Error => err
+      $stderr.puts err.to_s.capitalize
+      exit 1
+    end
+    new(options).perform
+  end
+
+  attr_reader :options
+
+  def initialize(options)
+    super()
+
+    @options = options
+    options.host ||= 'localhost'
+    options.prefix ||= 'smf_'
+
+    begin
+      timezone = `php -i`.lines.each do |line|
+        key, *vals = line.split(' => ').map(&:strip)
+        break vals[0] if key == 'Default timezone'
+      end
+      Time.zone = timezone
+    rescue Errno::ENOENT
+      $stderr.puts "Cannot autodetect PHP timezone setting, php not found in $PATH"
+    rescue ArgumentError
+      $stderr.puts "Cannot set timezone '#{timezone}' (from PHP)"
+    end
+
+    if options.database.blank?
+      $stderr.puts "No database name given."
+      $stderr.puts options.usage
+      exit 1
+    end
+    if options.password == :ask
+      require 'highline'
+      $stderr.print "Enter password for MySQL database `#{options.database}`: "
+      options.password = HighLine.new.ask('') {|q| q.echo = false }
+    end
+
+    @db = Mysql2::Client.new(host: options.host, username: options.username,
+      password: options.password, database: options.database)
+  end
+
+  def execute
+    import_groups
+    import_users
+    import_categories
+    import_posts
+  end
+
+  def import_groups
+    puts '', 'creating groups'
+
+    total = query(<<-SQL, as: :single)
+      SELECT COUNT(*) FROM {prefix}membergroups
+      WHERE min_posts = -1 AND group_type IN (1, 2)
+    SQL
+
+    create_groups(query(<<-SQL), total: total) {|group| group }
+      SELECT id_group AS id, group_name AS name
+      FROM {prefix}membergroups
+      WHERE min_posts = -1 AND group_type IN (1, 2)
+    SQL
+  end
+
+  GUEST_GROUP = -1
+  MEMBER_GROUP = 0
+  ADMIN_GROUP = 1
+  MODERATORS_GROUP = 2
+
+  def import_users
+    puts '', 'creating users'
+    total = query("SELECT COUNT(*) FROM {prefix}members", as: :single)
+
+    create_users(query(<<-SQL), total: total) do |member|
+      SELECT a.id_member, a.member_name, a.date_registered, a.real_name, a.email_address,
+             a.is_activated, a.last_login, a.birthdate, a.member_ip, a.id_group, a.additional_groups,
+             b.id_attach, b.file_hash, b.filename
+      FROM {prefix}members AS a
+      LEFT JOIN {prefix}attachments AS b ON a.id_member = b.id_member
+    SQL
+      group_ids = [ member[:id_group], *member[:additional_groups].split(',').map(&:to_i) ]
+      {
+        id: member[:id_member],
+        username: member[:member_name],
+        created_at: Time.zone.at(member[:date_registered]),
+        name: member[:real_name],
+        email: member[:email_address],
+        active: member[:is_activated] == 1,
+        approved: member[:is_activated] == 1,
+        last_seen_at: Time.zone.at(member[:last_login]),
+        date_of_birth: member[:birthdate],
+        ip_address: IPAddr.new(member[:member_ip]),
+        admin: group_ids.include?(ADMIN_GROUP),
+        moderator: group_ids.include?(MODERATORS_GROUP),
+        post_create_action: proc do |user|
+          GroupUser.transaction do
+            group_ids.each do |gid|
+              group_id = group_id_from_imported_group_id(gid) and
+                GroupUser.find_or_create_by(user: user, group_id: group_id)
+            end
+          end
+          if options.smfroot and member[:id_attach].present? and user.uploaded_avatar_id.blank?
+            path = find_smf_attachment_path(member[:id_attach], member[:file_hash], member[:filename]) and begin
+              upload = create_upload(user.id, path, member[:filename])
+              if upload.persisted?
+                user.update(uploaded_avatar_id: upload.id)
+              end
+            rescue SystemCallError => err
+              puts "Could not import avatar: #{err.message}"
+            end
+          end
+        end
+      }
+    end
+  end
+
+  def import_categories
+    create_categories(query(<<-SQL)) do |board|
+      SELECT id_board, id_parent, name, description, member_groups
+      FROM {prefix}boards
+      ORDER BY id_parent ASC, id_board ASC
+    SQL
+      parent_id = category_from_imported_category_id(board[:id_parent]).id if board[:id_parent] > 0
+      groups = (board[:member_groups] || "").split(/,/).map(&:to_i)
+      restricted = !groups.include?(GUEST_GROUP) && !groups.include?(MEMBER_GROUP)
+      {
+        id: board[:id_board],
+        name: board[:name],
+        description: board[:description],
+        parent_category_id: parent_id,
+        post_create_action: restricted && proc do |category|
+          category.update(read_restricted: true)
+          groups.each do |imported_group_id|
+            group_id = group_id_from_imported_group_id(imported_group_id) and
+            CategoryGroup.find_or_create_by(category: category, group_id: group_id) do |cg|
+              cg.permission_type = CategoryGroup.permission_types[:full]
+            end
+          end
+        end,
+      }
+    end
+  end
+
+  def import_posts
+    puts '', 'creating posts'
+    spinner = %w(/ - \\ |).cycle
+    total = query("SELECT COUNT(*) FROM {prefix}messages", as: :single)
+    PostCreator.class_eval do
+      def guardian
+        @guardian ||= if opts[:import_mode]
+          @@system_guardian ||= Guardian.new(Discourse.system_user)
+        else
+          Guardian.new(@user)
+        end
+      end
+    end
+
+    topics = Enumerator.new do |y|
+      last_topic_id = nil
+      topic_messages = nil
+      query("SELECT id_msg, id_topic, body FROM {prefix}messages ORDER BY id_topic ASC, poster_time ASC") do |message|
+        if last_topic_id != message[:id_topic]
+          y << topic_messages
+          last_topic_id = message[:id_topic]
+          topic_messages = [ message ]
+        else
+          topic_messages << message
+        end
+      end
+      y << topic_messages
+    end
+
+    graph = MessageDependencyGraph.new
+    topics.each do |messages|
+      next unless messages.present?
+      (messages.reverse << nil).each_cons(2) do |message, prev|
+        graph.add_message(message[:id_msg], prev ? prev[:id_msg] : nil,
+          extract_quoted_message_ids(message[:body]).to_a)
+      end
+      print "\r#{spinner.next}"
+    end
+
+    begin
+      cycles = graph.cycles
+      print "\r#{spinner.next}"
+      cycles.each do |cycle|
+        candidate = cycle.detect {|n| ((cycle - [n]) & n.quoted).present? }
+        candidate.ignore_quotes = true
+      end
+    end while cycles.present?
+    message_order = graph.tsort
+    print "\r#{spinner.next}"
+
+    query(<<-SQL, as: :array)
+      CREATE TEMPORARY TABLE {prefix}import_message_order (
+        message_id int(11) NOT NULL,
+        message_order int(11) NOT NULL AUTO_INCREMENT,
+        ignore_quotes tinyint(1) NOT NULL,
+        PRIMARY KEY (message_id),
+        UNIQUE KEY message_order (message_order)
+      ) ENGINE=MEMORY
+    SQL
+    message_order.each_slice(100) do |nodes|
+      query(<<-SQL, as: :array)
+        INSERT INTO {prefix}import_message_order (message_id, ignore_quotes)
+        VALUES #{ nodes.map {|n| "(#{n.id}, #{n.ignore_quotes? ? 1 : 0})" }.join(',') }
+      SQL
+      print "\r#{spinner.next}"
+    end
+
+    postprocess = []
+    create_posts(query(<<-SQL), total: total) do |message|
+      SELECT m.id_msg, m.id_topic, m.id_member, m.poster_time, m.body, o.ignore_quotes,
+             m.subject, t.id_board, t.id_first_msg
+      FROM {prefix}messages AS m
+      LEFT JOIN {prefix}import_message_order AS o ON o.message_id = m.id_msg
+      LEFT JOIN {prefix}topics AS t ON t.id_topic = m.id_topic
+      ORDER BY o.message_order ASC
+    SQL
+      skip = false
+      post = {
+        id: message[:id_msg],
+        user_id: user_id_from_imported_user_id(message[:id_member]) || -1,
+        raw: convert_message_body(message[:body], ignore_quotes: message[:ignore_quotes] == 1),
+        created_at: Time.zone.at(message[:poster_time]),
+        post_create_action: proc {|post| postprocess << post }
+      }
+      if message[:id_msg] == message[:id_first_msg]
+        post[:category] = category_from_imported_category_id(message[:id_board]).try(:name)
+        post[:title] = decode_entities(message[:subject])
+      else
+        parent = topic_lookup_from_imported_post_id(message[:id_first_msg])
+        if parent
+          post[:topic_id] = parent[:topic_id]
+        else
+          puts "Parent post #{message[:id_first_msg]} doesn't exist. Skipping #{message[:id_msg]}: #{message[:subject][0..40]}"
+          skip = true
+        end
+      end
+      skip ? nil : post
+    end
+
+    postprocess.each do |post|
+      post.raw = convert_quotes(post.raw)
+      post.rebake!
+      post.save!(validate: false)
+    end
+  end
+
+  private
+
+  def query(sql, **opts, &block)
+    return __query(sql).to_a                       if opts[:as] == :array
+    return __query(sql, as: :array).first[0]       if opts[:as] == :single
+    return __query(sql, stream: true).each(&block) if block_given?
+    return __query(sql, stream: true)
+  end
+
+  def __query(sql, **opts)
+    @db.query(sql.gsub('{prefix}', options.prefix),
+      {symbolize_keys: true, cache_rows: false}.merge(opts))
+  end
+
+  def find_smf_attachment_path(attachment_id, file_hash, filename)
+    [ filename, "#{attachment_id}_#{file_hash}" ]
+      .map {|name| File.join(options.smfroot, 'attachments', name) }
+      .detect {|file| File.exists?(file) }
+  end
+
+  def decode_entities(*args)
+    (@html_entities ||= HTMLEntities.new).decode(*args)
+  end
+
+  def convert_message_body(body, attachments = [], **opts)
+    body = decode_entities(body.gsub(/<br\s*\/>/, "\n"))
+    body.gsub!(ColorPattern, '\k<inner>')
+    body.gsub!(ListPattern) do |s|
+      params = parse_tag_params($~[:params])
+      tag = params['type'] == 'decimal' ? 'ol' : 'ul'
+      "\n[#{tag}]#{$~[:inner].strip}[/#{tag}]\n"
+    end
+    body.gsub!(XListPattern) do |s|
+      r = "\n[ul]"
+      s.lines.each {|l| r << '[li]' << l.strip.sub(/^\[x\]\s*/, '') << '[/li]' }
+      r << "[/ul]\n"
+    end
+    # TODO: attachments
+    return opts[:ignore_quotes] ? body : convert_quotes(body)
+  end
+
+  def convert_quotes(body)
+    body.to_s.gsub(QuotePattern) do |s|
+      inner = $~[:inner].strip
+      params = parse_tag_params($~[:params])
+      if params['author'].present?
+        quote = "[quote=\"#{params['author']}"
+        if QuoteParamsPattern =~ params['link']
+          tl = topic_lookup_from_imported_post_id($~[:msg].to_i)
+          quote << ", post:#{tl[:post_number]}, topic:#{tl[:topic_id]}"
+        end
+        quote << "\"]#{inner}[/quote]"
+      else
+        "<blockquote>#{inner}</blockquote>"
+      end
+    end
+  end
+
+  def extract_quoted_message_ids(body)
+    Set.new.tap do |quoted|
+      body.scan(/\[quote\s+([^\]]+)\s*\]/) do |params|
+        params = parse_tag_params(params)
+        if params.has_key?("link")
+          match = QuoteParamsPattern.match(params["link"])
+          quoted << match[:msg].to_i if match
+        end
+      end
+    end
+  end
+
+  # param1=value1=still1 value1 param2=value2 ...
+  # => {'param1' => 'value1=still1 value1', 'param2' => 'value2 ...'}
+  def parse_tag_params(params)
+    params.to_s.strip.scan(/(?<param>\w+)=(?<value>(?:(?>\S+)|\s+(?!\w+=))*)/).to_h
+  end
+
+  class << self
+    private
+
+    # [tag param=value param2=value2]
+    #   text
+    #   [tag nested=true]text[/tag]
+    # [/tag]
+    # => match[:params] == 'param=value param2=value2'
+    #    match[:inner] == "\n  text\n  [tag nested=true]text[/tag]\n"
+    def build_nested_tag_regex(ltag, rtag = nil)
+      rtag ||= '/'+ltag
+      %r{
+        \[#{ltag}(?-x:[ =](?<params>[^\]]*))?\]            # consume open tag, followed by...
+          (?<inner>(?:
+            (?> [^\[]+ )                                   # non-tags, or...
+            |
+            \[(?! #{ltag}(?-x:[ =][^\]]*)?\] | #{rtag}\])  # different tags, or ...
+            |
+            (?<re>                                         # recursively matched tags of the same kind
+              \[#{ltag}(?-x:[ =][^\]]*)?\]
+                (?:
+                  (?> [^\[]+ )
+                  |
+                  \[(?! #{ltag}(?-x:[ =][^\]]*)?\] | #{rtag}\])
+                  |
+                  \g<re>                                   # recursion here
+                )*
+              \[#{rtag}\]
+            )
+          )*)
+        \[#{rtag}\]
+      }x
+    end
+  end
+
+  QuoteParamsPattern = /^topic=(?<topic>\d+).msg(?<msg>\d+)#msg\k<msg>$/
+  XListPattern = /(?<xblock>(?>^\[x\]\s*(?<line>.*)$\n?)+)/
+  QuotePattern = build_nested_tag_regex('quote')
+  ColorPattern = build_nested_tag_regex('color')
+  ListPattern = build_nested_tag_regex('list')
+
+  # Provides command line options and parses the SMF settings file.
+  class Options
+
+    class Error < StandardError ; end
+
+    def self.parse!(args = ARGV)
+      new.tap {|options| options.parse!(args) }
+    end
+
+    def parse!(args = ARGV)
+      begin
+        parser.parse!(args)
+      rescue OptionParser::ParseError => err
+        raise Error, err.message
+      end
+      raise Error, 'too many arguments' if args.length > 1
+      self.smfroot = args.first
+      read_smf_settings if self.smfroot
+    end
+
+    def usage
+      parser.to_s
+    end
+
+    attr_accessor :host
+    attr_accessor :username
+    attr_accessor :password
+    attr_accessor :database
+    attr_accessor :prefix
+    attr_accessor :smfroot
+
+    private
+
+    def read_smf_settings
+      settings = File.join(self.smfroot, 'Settings.php')
+      IO.readlines(settings).each do |line|
+        next unless m = /\$([a-z_]+)\s*=\s*['"](.+?)['"]\s*;\s*((#|\/\/).*)?$/.match(line)
+        case m[1]
+        when 'db_server' then self.host ||= m[2]
+        when 'db_user'   then self.username ||= m[2]
+        when 'db_passwd' then self.password ||= m[2]
+        when 'db_name'   then self.database ||= m[2]
+        when 'db_prefix' then self.prefix ||= m[2]
+        end
+      end
+    rescue => err
+      raise Error, "#{settings}: failed reading SMF settings, #{err.message}"
+    end
+
+    def parser
+      @parser ||= OptionParser.new do |o|
+        o.banner = "Usage:\t#{File.basename($0)} [options] [SMFROOT]"
+        o.on('-h', '--host=HOST', 'MySQL server hostname') {|s| self.host = s }
+        o.on('-u', '--username=USER', 'MySQL username') {|s| self.username = s }
+        o.on('-p', '--password[=PASS]', 'MySQL password') {|s| self.password = s || :ask }
+        o.on('-d', '--database=DB', 'Name of SMF database') {|s| self.database = s }
+        o.on('-f', '--prefix=PREFIX', 'Table names prefix') {|s| self.prefix = s }
+      end
+    end
+
+  end #Options
+
+
+  # Framework around TSort, used to build a dependency graph over messages
+  # to find and solve cyclic quotations.
+  class MessageDependencyGraph
+    include TSort
+
+    def initialize
+      @nodes = {}
+    end
+
+    def [](key)
+      @nodes[key]
+    end
+
+    def add_message(id, prev = nil, quoted = [])
+      @nodes[id] = Node.new(self, id, prev, quoted)
+    end
+
+    def tsort_each_node(&block)
+      @nodes.each_value(&block)
+    end
+
+    def tsort_each_child(node, &block)
+      node.dependencies.each(&block)
+    end
+
+    def cycles
+      strongly_connected_components.select {|c| c.length > 1 }.to_a
+    end
+
+
+    class Node
+      attr_reader :id
+
+      def initialize(graph, id, prev = nil, quoted = [])
+        @graph = graph
+        @id = id
+        @prev = prev
+        @quoted = quoted
+      end
+
+      def prev
+        @graph[@prev]
+      end
+
+      def quoted
+        @quoted.map {|id| @graph[id] }
+      end
+
+      def ignore_quotes?
+        !!@ignore_quotes
+      end
+
+      def ignore_quotes=(value)
+        @ignore_quotes = !!value
+        @dependencies = nil
+      end
+
+      def dependencies
+        @dependencies ||= Set.new.tap do |deps|
+          deps.merge(quoted) unless ignore_quotes?
+          deps << prev if prev.present?
+        end.to_a
+      end
+
+      def hash
+        @id.hash
+      end
+
+      def eql?(other)
+        @id.eql?(other)
+      end
+
+      def inspect
+        "#<#{self.class.name}: id=#{id.inspect}, prev=#{prev.try(:id).inspect}, quoted=#{quoted.map{|e|e.id}.inspect}>"
+      end
+    end #Node
+
+  end #MessageDependencyGraph
+
+end
+
+ImportScripts::Smf2.run