Import optimizations for topic creation: Prevent queuing of most jobs when importing posts and topics. Only do some recalculations at the end of the import.

app/models/topic.rb

@@ -109,6 +109,7 @@ class Topic < ActiveRecord::Base
   attr_accessor :topic_list
   attr_accessor :meta_data
   attr_accessor :include_last_poster
+  attr_accessor :import_mode # set to true to optimize creation and save for imports
 
   # The regular order
   scope :topic_list_order, -> { order('topics.bumped_at desc') }
@@ -463,9 +464,9 @@ class Topic < ActiveRecord::Base
       end
 
       if success
-        CategoryFeaturedTopic.feature_topics_for(old_category)
+        CategoryFeaturedTopic.feature_topics_for(old_category) unless @import_mode
         Category.where(id: cat.id).update_all 'topic_count = topic_count + 1'
-        CategoryFeaturedTopic.feature_topics_for(cat) unless old_category.try(:id) == cat.try(:id)
+        CategoryFeaturedTopic.feature_topics_for(cat) unless @import_mode || old_category.try(:id) == cat.try(:id)
       else
         return false
       end

lib/post_creator.rb

@@ -74,9 +74,9 @@ class PostCreator
     end
 
     if @post
-      PostAlerter.post_created(@post)
+      PostAlerter.post_created(@post) unless @opts[:import_mode]
 
-      handle_spam
+      handle_spam unless @opts[:import_mode]
       track_latest_on_category
       enqueue_jobs
     end
@@ -233,6 +233,7 @@ class PostCreator
   end
 
   def consider_clearing_flags
+    return if @opts[:import_mode]
     return unless @topic.private_message? && @post.post_number > 1 && @topic.user_id != @post.user_id
 
     clear_possible_flags(@topic)
@@ -240,7 +241,7 @@ class PostCreator
 
   def update_user_counts
     # We don't count replies to your own topics
-    if @user.id != @topic.user_id
+    if !@opts[:import_mode] && @user.id != @topic.user_id
       @user.user_stat.update_topic_reply_count
       @user.user_stat.save!
     end
@@ -250,6 +251,7 @@ class PostCreator
   end
 
   def publish
+    return if @opts[:import_mode]
     return unless @post.post_number > 1
 
     MessageBus.publish("/topic/#{@post.topic_id}",{
@@ -288,7 +290,7 @@ class PostCreator
 
   def enqueue_jobs
     return unless @post && !@post.errors.present?
-    PostJobsEnqueuer.new(@post, @topic, new_topic?).enqueue_jobs
+    PostJobsEnqueuer.new(@post, @topic, new_topic?, {import_mode: @opts[:import_mode]}).enqueue_jobs
   end
 
   def new_topic?

lib/post_jobs_enqueuer.rb

@@ -1,14 +1,15 @@
 class PostJobsEnqueuer
-  def initialize(post, topic, new_topic)
+  def initialize(post, topic, new_topic, opts={})
     @post = post
     @topic = topic
     @new_topic = new_topic
+    @opts = opts
   end
 
   def enqueue_jobs
     # We need to enqueue jobs after the transaction. Otherwise they might begin before the data has
     # been comitted.
-    feature_topic_users
+    feature_topic_users unless @opts[:import_mode]
     trigger_post_post_process
     unless skip_after_create?
       after_post_create
@@ -51,6 +52,6 @@ class PostJobsEnqueuer
   end
 
   def skip_after_create?
-    @topic.private_message? || @post.post_type == Post.types[:moderator_action]
+    @opts[:import_mode] || @topic.private_message? || @post.post_type == Post.types[:moderator_action]
   end
 end

lib/topic_creator.rb

@@ -48,7 +48,7 @@ class TopicCreator
       last_post_user_id: @user.id
     }
 
-    [:subtype, :archetype, :meta_data].each do |key|
+    [:subtype, :archetype, :meta_data, :import_mode].each do |key|
       topic_params[key] = @opts[key] if @opts[key].present?
     end
 

script/import_scripts/base.rb

@@ -58,6 +58,8 @@ class ImportScripts::Base
 
     update_bumped_at
     update_feature_topic_users
+    update_category_featured_topics
+    update_topic_count_replies
 
     puts '', 'Done'
 
@@ -266,6 +268,7 @@ class ImportScripts::Base
   def create_post(opts, import_id)
     user = User.find(opts[:user_id])
     opts = opts.merge(skip_validations: true)
+    opts[:import_mode] = true
     opts[:custom_fields] ||= {}
     opts[:custom_fields]['import_id'] = import_id
 
@@ -292,11 +295,12 @@ class ImportScripts::Base
   end
 
   def update_bumped_at
+    puts '', "updating bumped_at on topics"
     Post.exec_sql("update topics t set bumped_at = (select max(created_at) from posts where topic_id = t.id and post_type != #{Post.types[:moderator_action]})")
   end
 
   def update_feature_topic_users
-    puts '', "updating featured topic users"
+    puts "updating featured topic users"
 
     total_count = Topic.count
     progress_count = 0
@@ -308,6 +312,27 @@ class ImportScripts::Base
     end
   end
 
+  def update_category_featured_topics
+    puts '', "updating featured topics in categories"
+    Category.find_each do |category|
+      CategoryFeaturedTopic.feature_topics_for(category)
+    end
+  end
+
+  def update_topic_count_replies
+    puts "updating user topic reply counts"
+
+    total_count = User.real.count
+    progress_count = 0
+
+    User.real.find_each do |u|
+      u.user_stat.update_topic_reply_count
+      u.user_stat.save!
+      progress_count += 1
+      print_status(progress_count, total_count)
+    end
+  end
+
   def print_status(current, max)
     print "\r%9d / %d (%5.1f%%)    " % [current, max, ((current.to_f / max.to_f) * 100).round(1)]
   end