FIX & PERF: vanilla import

PERF: disabled refresh_avatar callback when importing users
PERF: avoid using UsernameSuggester when not needed
FIX: categories wasn't working
FIX: posts from deleted users are now from the system user

app/models/user.rb

@@ -67,19 +67,20 @@ class User < ActiveRecord::Base
   validate :password_validator
   validates :ip_address, allowed_ip_address: {on: :create, message: :signup_not_allowed}
 
-  before_save :update_username_lower
-  before_save :ensure_password_is_hashed
   after_initialize :add_trust_level
   after_initialize :set_default_email_digest
   after_initialize :set_default_external_links_in_new_tab
 
-  after_save :update_tracked_topics
-  after_save :clear_global_notice_if_needed
-
   after_create :create_email_token
   after_create :create_user_stat
   after_create :create_user_profile
   after_create :ensure_in_trust_level_group
+
+  before_save :update_username_lower
+  before_save :ensure_password_is_hashed
+
+  after_save :update_tracked_topics
+  after_save :clear_global_notice_if_needed
   after_save :refresh_avatar
   after_save :badge_grant
 
@@ -95,6 +96,9 @@ class User < ActiveRecord::Base
   # This is just used to pass some information into the serializer
   attr_accessor :notification_channel_position
 
+  # set to true to optimize creation and save for imports
+  attr_accessor :import_mode
+
   scope :blocked, -> { where(blocked: true) } # no index
   scope :not_blocked, -> { where(blocked: false) } # no index
   scope :suspended, -> { where('suspended_till IS NOT NULL AND suspended_till > ?', Time.zone.now) } # no index
@@ -594,6 +598,8 @@ class User < ActiveRecord::Base
   end
 
   def refresh_avatar
+    return if @import_mode
+
     avatar = user_avatar || create_user_avatar
     gravatar_downloaded = false
 

script/import_scripts/base.rb

@@ -17,6 +17,7 @@ class ImportScripts::Base
 
   def initialize
     require File.expand_path(File.dirname(__FILE__) + "/../../config/environment")
+    preload_i18n
 
     @bbcode_to_md = true if ARGV.include?('bbcode-to-md')
     @existing_groups = {}
@@ -48,6 +49,11 @@ class ImportScripts::Base
     end
   end
 
+  def preload_i18n
+    I18n.t("test")
+    ActiveSupport::Inflector.transliterate("test")
+  end
+
   def perform
     Rails.logger.level = 3 # :error, so that we don't create log files that are many GB
 
@@ -62,12 +68,14 @@ class ImportScripts::Base
 
     execute
 
+    puts ""
+
     update_bumped_at
     update_feature_topic_users
     update_category_featured_topics
     update_topic_count_replies
 
-    puts '', 'Done'
+    puts "", "Done"
 
   ensure
     RateLimiter.enable
@@ -132,8 +140,6 @@ class ImportScripts::Base
   # group in the original datasource. The given id will not be used
   # to create the Discourse group record.
   def create_groups(results, opts={})
-    puts "", "creating groups"
-
     groups_created = 0
     groups_skipped = 0
     total = opts[:total] || results.size
@@ -182,8 +188,6 @@ class ImportScripts::Base
   # user in the original datasource. The given id will not be used to
   # create the Discourse user record.
   def create_users(results, opts={})
-    puts "", "creating users"
-
     num_users_before = User.count
     users_created = 0
     users_skipped = 0
@@ -225,17 +229,21 @@ class ImportScripts::Base
     opts.delete(:id)
     post_create_action = opts.delete(:post_create_action)
     existing = User.where(email: opts[:email].downcase, username: opts[:username]).first
-    return existing if existing and existing.custom_fields["import_id"].to_i == import_id.to_i
+    return existing if existing && existing.custom_fields["import_id"].to_i == import_id.to_i
 
     bio_raw = opts.delete(:bio_raw)
-    opts[:name] = User.suggest_name(opts[:name]) if opts[:name]
-    opts[:username] = UserNameSuggester.suggest((opts[:username].present? ? opts[:username] : nil) || opts[:name] || opts[:email])
+    opts[:name] = User.suggest_name(opts[:email]) unless opts[:name]
+    if opts[:username].blank? || !User.username_available?(opts[:username])
+      opts[:username] = UserNameSuggester.suggest(opts[:username] || opts[:name] || opts[:email])
+    end
     opts[:email] = opts[:email].downcase
     opts[:trust_level] = TrustLevel.levels[:basic] unless opts[:trust_level]
+    opts[:import_mode] = true
 
     u = User.new(opts)
     u.custom_fields["import_id"] = import_id
     u.custom_fields["import_username"] = opts[:username] if opts[:username].present?
+    u.custom_fields["import_avatar_url"] = opts[:avatar_url] if opts[:avatar_url].present?
 
     begin
       User.transaction do
@@ -266,8 +274,6 @@ class ImportScripts::Base
   # create the Discourse category record.
   # Optional attributes are position, description, and parent_category_id.
   def create_categories(results)
-    puts "", "creating categories"
-
     results.each do |c|
       params = yield(c)
       puts "    #{params[:name]}"
@@ -308,8 +314,6 @@ class ImportScripts::Base
   # Topics should give attributes title and category.
   # Replies should provide topic_id. Use topic_lookup_from_imported_post_id to find the topic.
   def create_posts(results, opts={})
-    puts "", "creating posts"
-
     skipped = 0
     created = 0
     total = opts[:total] || results.size
@@ -389,8 +393,8 @@ class ImportScripts::Base
   end
 
   def close_inactive_topics(opts={})
+    puts "", "Closing topics that have been inactive for more than #{num_days} days."
     num_days = opts[:days] || 30
-    puts '', "Closing topics that have been inactive for more than #{num_days} days."
 
     query = Topic.where('last_posted_at < ?', num_days.days.ago).where(closed: false)
     total_count = query.count
@@ -404,7 +408,7 @@ class ImportScripts::Base
   end
 
   def update_bumped_at
-    puts '', "updating bumped_at on topics"
+    puts "updating bumped_at on topics"
     Post.exec_sql("update topics t set bumped_at = (select max(created_at) from posts where topic_id = t.id and post_type != #{Post.types[:moderator_action]})")
   end
 
@@ -422,7 +426,7 @@ class ImportScripts::Base
   end
 
   def update_category_featured_topics
-    puts '', "updating featured topics in categories"
+    puts "updating featured topics in categories"
     Category.find_each do |category|
       CategoryFeaturedTopic.feature_topics_for(category)
     end

script/import_scripts/vanilla.rb

@@ -79,15 +79,22 @@ class ImportScripts::Vanilla < ImportScripts::Base
       create_users(@users) do |user|
         next if user[:name] == "[Deleted User]"
 
-        {
+        u = {
           id: user[:user_id],
           email: user[:email],
           name: user[:name],
           created_at: parse_date(user[:date_inserted]),
           bio_raw: clean_up(user[:discovery_text]),
+          avatar_url: user[:photo],
           moderator: @user_roles.select { |ur| ur[:user_id] == user[:user_id] }.map { |ur| ur[:role_id] }.include?(moderator_role_id),
           admin: @user_roles.select { |ur| ur[:user_id] == user[:user_id] }.map { |ur| ur[:role_id] }.include?(admin_role_id),
         }
+
+        # if @comments.select { |c| c[:insert_user_id] == user[:user_id] }.map { |c| c[:discussion_id] }.uniq.count > 3
+        #   u[:trust_level] = TrustLevel.levels[:regular]
+        # end
+
+        u
       end
     end
 
@@ -118,14 +125,14 @@ class ImportScripts::Vanilla < ImportScripts::Base
       c = {
         id: category[:category_id],
         name: category[:name],
-        user_id: user_id_from_imported_user_id(category[:insert_user_id]),
+        user_id: user_id_from_imported_user_id(category[:insert_user_id]) || Discourse::SYSTEM_USER_ID,
         position: category[:sort].to_i,
         created_at: parse_category_date(category[:date_inserted]),
         description: clean_up(category[:description]),
       }
       if category[:parent_category_id] != "-1"
-        parent_category = category_from_imported_category_id(category[:parent_category_id].to_i)
-        c[:parent_category_id] = parent_category[:id] if parent_category
+        parent_category = category_from_imported_category_id(category[:parent_category_id])
+        c[:parent_category_id] = parent_category.id if parent_category
       end
       c
     end
@@ -140,9 +147,9 @@ class ImportScripts::Vanilla < ImportScripts::Base
       create_posts(@discussions) do |discussion|
         {
           id: "discussion#" + discussion[:discussion_id],
-          user_id: user_id_from_imported_user_id(discussion[:insert_user_id]),
+          user_id: user_id_from_imported_user_id(discussion[:insert_user_id]) || Discourse::SYSTEM_USER_ID,
           title: discussion[:name],
-          category_id: category_from_imported_category_id(discussion[:category_id]).try(:id),
+          category: category_from_imported_category_id(discussion[:category_id]).try(:name),
           raw: clean_up(discussion[:body]),
           created_at: parse_date(discussion[:date_inserted]),
         }
@@ -157,7 +164,7 @@ class ImportScripts::Vanilla < ImportScripts::Base
 
         {
           id: "comment#" + comment[:comment_id],
-          user_id: user_id_from_imported_user_id(comment[:insert_user_id]),
+          user_id: user_id_from_imported_user_id(comment[:insert_user_id]) || Discourse::SYSTEM_USER_ID,
           topic_id: t[:topic_id],
           raw: clean_up(comment[:body]),
           created_at: parse_date(comment[:date_inserted]),
@@ -207,7 +214,7 @@ class ImportScripts::Vanilla < ImportScripts::Base
         {
           archetype: Archetype.private_message,
           id: "message#" + message[:message_id],
-          user_id: user_id_from_imported_user_id(message[:insert_user_id]),
+          user_id: user_id_from_imported_user_id(message[:insert_user_id]) || Discourse::SYSTEM_USER_ID,
           topic_id: t[:topic_id],
           raw: clean_up(message[:body]),
           created_at: parse_date(message[:date_inserted]),
@@ -225,6 +232,7 @@ class ImportScripts::Vanilla < ImportScripts::Base
                  .gsub(/<\/?code\s*>/i, "`")
                  .gsub("&lt;", "<")
                  .gsub("&gt;", ">")
+                 # .gsub("*", "\\*")
     end
 
 end