From 7d5d5862c1c67a98059faacf23b710d4f08447d8 Mon Sep 17 00:00:00 2001 From: Neil Lalonde Date: Thu, 3 Jul 2014 14:43:24 -0400 Subject: [PATCH] Import optimizations for topic creation: Prevent queuing of most jobs when importing posts and topics. Only do some recalculations at the end of the import. --- app/models/topic.rb | 5 +++-- lib/post_creator.rb | 10 ++++++---- lib/post_jobs_enqueuer.rb | 7 ++++--- lib/topic_creator.rb | 2 +- script/import_scripts/base.rb | 27 ++++++++++++++++++++++++++- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/app/models/topic.rb b/app/models/topic.rb index 5410d91d7..dbd772feb 100644 --- a/app/models/topic.rb +++ b/app/models/topic.rb @@ -109,6 +109,7 @@ class Topic < ActiveRecord::Base attr_accessor :topic_list attr_accessor :meta_data attr_accessor :include_last_poster + attr_accessor :import_mode # set to true to optimize creation and save for imports # The regular order scope :topic_list_order, -> { order('topics.bumped_at desc') } @@ -463,9 +464,9 @@ class Topic < ActiveRecord::Base end if success - CategoryFeaturedTopic.feature_topics_for(old_category) + CategoryFeaturedTopic.feature_topics_for(old_category) unless @import_mode Category.where(id: cat.id).update_all 'topic_count = topic_count + 1' - CategoryFeaturedTopic.feature_topics_for(cat) unless old_category.try(:id) == cat.try(:id) + CategoryFeaturedTopic.feature_topics_for(cat) unless @import_mode || old_category.try(:id) == cat.try(:id) else return false end diff --git a/lib/post_creator.rb b/lib/post_creator.rb index 69b9240c6..e98913bdc 100644 --- a/lib/post_creator.rb +++ b/lib/post_creator.rb @@ -74,9 +74,9 @@ class PostCreator end if @post - PostAlerter.post_created(@post) + PostAlerter.post_created(@post) unless @opts[:import_mode] - handle_spam + handle_spam unless @opts[:import_mode] track_latest_on_category enqueue_jobs end @@ -233,6 +233,7 @@ class PostCreator end def consider_clearing_flags + return if @opts[:import_mode] return unless @topic.private_message? && @post.post_number > 1 && @topic.user_id != @post.user_id clear_possible_flags(@topic) @@ -240,7 +241,7 @@ class PostCreator def update_user_counts # We don't count replies to your own topics - if @user.id != @topic.user_id + if !@opts[:import_mode] && @user.id != @topic.user_id @user.user_stat.update_topic_reply_count @user.user_stat.save! end @@ -250,6 +251,7 @@ class PostCreator end def publish + return if @opts[:import_mode] return unless @post.post_number > 1 MessageBus.publish("/topic/#{@post.topic_id}",{ @@ -288,7 +290,7 @@ class PostCreator def enqueue_jobs return unless @post && !@post.errors.present? - PostJobsEnqueuer.new(@post, @topic, new_topic?).enqueue_jobs + PostJobsEnqueuer.new(@post, @topic, new_topic?, {import_mode: @opts[:import_mode]}).enqueue_jobs end def new_topic? diff --git a/lib/post_jobs_enqueuer.rb b/lib/post_jobs_enqueuer.rb index e5abfc878..219ea863f 100644 --- a/lib/post_jobs_enqueuer.rb +++ b/lib/post_jobs_enqueuer.rb @@ -1,14 +1,15 @@ class PostJobsEnqueuer - def initialize(post, topic, new_topic) + def initialize(post, topic, new_topic, opts={}) @post = post @topic = topic @new_topic = new_topic + @opts = opts end def enqueue_jobs # We need to enqueue jobs after the transaction. Otherwise they might begin before the data has # been comitted. - feature_topic_users + feature_topic_users unless @opts[:import_mode] trigger_post_post_process unless skip_after_create? after_post_create @@ -51,6 +52,6 @@ class PostJobsEnqueuer end def skip_after_create? - @topic.private_message? || @post.post_type == Post.types[:moderator_action] + @opts[:import_mode] || @topic.private_message? || @post.post_type == Post.types[:moderator_action] end end diff --git a/lib/topic_creator.rb b/lib/topic_creator.rb index 62f3ee2f0..47f75cf67 100644 --- a/lib/topic_creator.rb +++ b/lib/topic_creator.rb @@ -48,7 +48,7 @@ class TopicCreator last_post_user_id: @user.id } - [:subtype, :archetype, :meta_data].each do |key| + [:subtype, :archetype, :meta_data, :import_mode].each do |key| topic_params[key] = @opts[key] if @opts[key].present? end diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb index da74bd5e4..c3278b3c2 100644 --- a/script/import_scripts/base.rb +++ b/script/import_scripts/base.rb @@ -58,6 +58,8 @@ class ImportScripts::Base update_bumped_at update_feature_topic_users + update_category_featured_topics + update_topic_count_replies puts '', 'Done' @@ -266,6 +268,7 @@ class ImportScripts::Base def create_post(opts, import_id) user = User.find(opts[:user_id]) opts = opts.merge(skip_validations: true) + opts[:import_mode] = true opts[:custom_fields] ||= {} opts[:custom_fields]['import_id'] = import_id @@ -292,11 +295,12 @@ class ImportScripts::Base end def update_bumped_at + puts '', "updating bumped_at on topics" Post.exec_sql("update topics t set bumped_at = (select max(created_at) from posts where topic_id = t.id and post_type != #{Post.types[:moderator_action]})") end def update_feature_topic_users - puts '', "updating featured topic users" + puts "updating featured topic users" total_count = Topic.count progress_count = 0 @@ -308,6 +312,27 @@ class ImportScripts::Base end end + def update_category_featured_topics + puts '', "updating featured topics in categories" + Category.find_each do |category| + CategoryFeaturedTopic.feature_topics_for(category) + end + end + + def update_topic_count_replies + puts "updating user topic reply counts" + + total_count = User.real.count + progress_count = 0 + + User.real.find_each do |u| + u.user_stat.update_topic_reply_count + u.user_stat.save! + progress_count += 1 + print_status(progress_count, total_count) + end + end + def print_status(current, max) print "\r%9d / %d (%5.1f%%) " % [current, max, ((current.to_f / max.to_f) * 100).round(1)] end