PERF: optimise and improve topic similarity search

FIX: shows up similarity search with blank results

app/assets/javascripts/discourse/controllers/composer-messages.js.es6

@@ -29,6 +29,15 @@ export default Ember.ArrayController.extend({
     **/
     closeMessage: function(message) {
       this.removeObject(message);
+    },
+
+    hideMessage: function(message) {
+      var messagesByTemplate = this.get('messagesByTemplate'),
+        templateName = message.get('templateName');
+
+      // kind of hacky but the visibility depends on this
+      messagesByTemplate[templateName] = undefined;
+      this.removeObject(message);
     }
   },
 

app/assets/javascripts/discourse/controllers/composer.js.es6

@@ -215,7 +215,9 @@ export default Discourse.Controller.extend({
     if (!this.get('model.creatingTopic')) return;
 
     var body = this.get('model.reply'),
-        title = this.get('model.title');
+        title = this.get('model.title'),
+        self = this,
+        message;
 
     // Ensure the fields are of the minimum length
     if (body.length < Discourse.SiteSettings.min_body_similar_length ||
@@ -229,11 +231,19 @@ export default Discourse.Controller.extend({
       similarTopics.pushObjects(newTopics);
 
       if (similarTopics.get('length') > 0) {
-        messageController.popup(Discourse.ComposerMessage.create({
+        message = Discourse.ComposerMessage.create({
           templateName: 'composer/similar_topics',
           similarTopics: similarTopics,
           extraClass: 'similar-topics'
-        }));
+        });
+
+        self.set('similarTopicsMessage', message);
+        messageController.popup(message);
+      } else {
+        message = self.get('similarTopicsMessage');
+        if (message) {
+          messageController.send('hideMessage', message);
+        }
       }
     });
 

app/models/topic.rb

@@ -363,21 +363,38 @@ class Topic < ActiveRecord::Base
     return [] unless title.present?
     return [] unless raw.present?
 
-    similar = Topic.select(sanitize_sql_array(["topics.*, similarity(topics.title, :title) + similarity(p.raw, :raw) AS similarity", title: title, raw: raw]))
+    filter_words = Search.prepare_data(title + " " + raw[0...200]);
-                     .visible
+    ts_query = Search.ts_query(filter_words, nil, "|")
-                     .where(closed: false, archived: false)
-                     .secured(Guardian.new(user))
-                     .listable_topics
-                     .joins("LEFT OUTER JOIN posts AS p ON p.topic_id = topics.id AND p.post_number = 1")
-                     .limit(SiteSetting.max_similar_results)
-                     .order('similarity desc')
 
     # Exclude category definitions from similar topic suggestions
+
+    candidates = Topic.visible
+       .secured(Guardian.new(user))
+       .listable_topics
+       .joins('JOIN posts p ON p.topic_id = topics.id AND p.post_number = 1')
+       .joins('JOIN post_search_data s ON p.id = s.post_id')
+       .where("search_data @@ #{ts_query}")
+       .order("ts_rank(search_data, #{ts_query}) DESC")
+       .limit(SiteSetting.max_similar_results * 3)
+
     exclude_topic_ids = Category.pluck(:topic_id).compact!
     if exclude_topic_ids.present?
-      similar = similar.where("topics.id NOT IN (?)", exclude_topic_ids)
+      candidates = candidates.where("topics.id NOT IN (?)", exclude_topic_ids)
     end
 
+    candidate_ids = candidates.pluck(:id)
+
+
+    return [] unless candidate_ids.present?
+
+
+    similar = Topic.select(sanitize_sql_array(["topics.*, similarity(topics.title, :title) + similarity(topics.title, :raw) AS similarity", title: title, raw: raw]))
+                     .joins("JOIN posts AS p ON p.topic_id = topics.id AND p.post_number = 1")
+                     .limit(SiteSetting.max_similar_results)
+                     .where("topics.id IN (?)", candidate_ids)
+                     .where("similarity(topics.title, :title) + similarity(topics.title, :raw) > 0.2", raw: raw, title: title)
+                     .order('similarity desc')
+
     similar
   end
 

lib/search.rb

@@ -240,10 +240,10 @@ class Search
       self.class.query_locale
     end
 
-    def self.ts_query(term, locale = nil)
+    def self.ts_query(term, locale = nil, joiner = "&")
       locale = Post.sanitize(locale) if locale
-      all_terms = term.gsub(/[:()&!'"]/,'').split
+      all_terms = term.gsub(/[*:()&!'"]/,'').squish.split
-      query = Post.sanitize(all_terms.map {|t| "#{PG::Connection.escape_string(t)}:*"}.join(" & "))
+      query = Post.sanitize(all_terms.map {|t| "#{PG::Connection.escape_string(t)}:*"}.join(" #{joiner} "))
       "TO_TSQUERY(#{locale || query_locale}, #{query})"
     end
 

spec/models/topic_spec.rb

@@ -211,7 +211,11 @@ describe Topic do
     end
 
     context 'with a similar topic' do
-      let!(:topic) { Fabricate(:topic, title: "Evil trout is the dude who posted this topic") }
+      let!(:topic) {
+        ActiveRecord::Base.observers.enable :search_observer
+        post = create_post(title: "Evil trout is the dude who posted this topic")
+        post.topic
+      }
 
       it 'returns the similar topic if the title is similar' do
         Topic.similar_to("has evil trout made any topics?", "i am wondering has evil trout made any topics?").should == [topic]