From 3ea477b17d310efe8b7d39e11d31fca2a82d7aa7 Mon Sep 17 00:00:00 2001 From: Neil Lalonde Date: Wed, 19 Mar 2014 10:14:05 -0400 Subject: [PATCH] FIX: performance of CategoryDetailedSerializer and Category.update_stats on large databases --- app/models/category.rb | 100 +++++++++++++----- ...318203559_add_created_at_index_to_posts.rb | 17 +++ 2 files changed, 91 insertions(+), 26 deletions(-) create mode 100644 db/migrate/20140318203559_add_created_at_index_to_posts.rb diff --git a/app/models/category.rb b/app/models/category.rb index 6b918dd0b..7d2a1cd3a 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -97,9 +97,53 @@ class Category < ActiveRecord::Base end end + def self.update_stats + topics_with_post_count = Topic + .select("topics.category_id, COUNT(*) topic_count, SUM(topics.posts_count) post_count") + .where("topics.id NOT IN (select cc.topic_id from categories cc WHERE topic_id IS NOT NULL)") + .group("topics.category_id") + .visible.to_sql + + Category.exec_sql < x.topic_count OR c.post_count <> x.post_count) + +SQL + + # Yes, there are a lot of queries happening below. + # Performing a lot of queries is actually faster than using one big update + # statement with sub-selects on large databases with many categories, + # topics, and posts. + # + # The old method with the one query is here: + # https://github.com/discourse/discourse/blob/5f34a621b5416a53a2e79a145e927fca7d5471e8/app/models/category.rb + # + # If you refactor this, test performance on a large database. + + Category.all.each do |c| + topics = c.topics.where(['topics.id <> ?', c.topic_id]).visible + c.topics_year = topics.created_since(1.year.ago).count + c.topics_month = topics.created_since(1.month.ago).count + c.topics_week = topics.created_since(1.week.ago).count + c.topics_day = topics.created_since(1.day.ago).count + + posts = c.visible_posts + c.posts_year = posts.created_since(1.year.ago).count + c.posts_month = posts.created_since(1.month.ago).count + c.posts_week = posts.created_since(1.week.ago).count + c.posts_day = posts.created_since(1.day.ago).count + + c.save if c.changed? + end + end + # Internal: Update category stats: # of topics and posts in past year, month, week for # all categories. - def self.update_stats + def self.update_stats_OLD topics = Topic .select("COUNT(*) topic_count") .where("topics.category_id = categories.id") @@ -115,6 +159,7 @@ class Category < ActiveRecord::Base topics_year = topics.created_since(1.year.ago).to_sql topics_month = topics.created_since(1.month.ago).to_sql topics_week = topics.created_since(1.week.ago).to_sql + topics_day = topics.created_since(1.day.ago).to_sql Category.exec_sql < ?', self.topic_id]) : query end - def topics_day - if val = $redis.get(topics_day_key) - val.to_i - else - val = self.topics.where(['topics.id <> ?', self.topic_id]).created_since(1.day.ago).visible.count - $redis.setex topics_day_key, 30.minutes.to_i, val - val - end - end + # def topics_day + # if val = $redis.get(topics_day_key) + # val.to_i + # else + # val = self.topics.where(['topics.id <> ?', self.topic_id]).created_since(1.day.ago).visible.count + # $redis.setex topics_day_key, 30.minutes.to_i, val + # val + # end + # end - def topics_day_key - "topics_day:cat-#{self.id}" - end + # def topics_day_key + # "topics_day:cat-#{self.id}" + # end - def posts_day - if val = $redis.get(posts_day_key) - val.to_i - else - val = self.visible_posts.created_since(1.day.ago).count - $redis.setex posts_day_key, 30.minutes.to_i, val - val - end - end + # def posts_day + # if val = $redis.get(posts_day_key) + # val.to_i + # else + # val = self.visible_posts.created_since(1.day.ago).count + # $redis.setex posts_day_key, 30.minutes.to_i, val + # val + # end + # end - def posts_day_key - "posts_day:cat-#{self.id}" - end + # def posts_day_key + # "posts_day:cat-#{self.id}" + # end # Internal: Generate the text of post prompting to enter category # description. diff --git a/db/migrate/20140318203559_add_created_at_index_to_posts.rb b/db/migrate/20140318203559_add_created_at_index_to_posts.rb new file mode 100644 index 000000000..5c6d9e661 --- /dev/null +++ b/db/migrate/20140318203559_add_created_at_index_to_posts.rb @@ -0,0 +1,17 @@ +class AddCreatedAtIndexToPosts < ActiveRecord::Migration + def up + execute "CREATE INDEX idx_posts_created_at_topic_id ON posts(created_at, topic_id) WHERE deleted_at IS NULL" + add_column :categories, :topics_day, :integer, default: 0 + add_column :categories, :posts_day, :integer, default: 0 + execute "DROP INDEX index_topics_on_deleted_at_and_visible_and_archetype_and_id" + add_index :topics, [:deleted_at, :visible, :archetype, :category_id, :id], name: "idx_topics_front_page" + end + + def down + execute "DROP INDEX idx_topics_front_page" + add_index :topics, [:deleted_at, :visible, :archetype, :id] + remove_column :categories, :posts_day + remove_column :categories, :topics_day + execute "DROP INDEX idx_posts_created_at_topic_id" + end +end