Refactor full-text search to get rid of tsvector columns.

Refactor full-text search on several tables (comments, dmails,
forum_posts, forum_topics, notes, and wiki_pages) to use to_tsvector
expression indexes instead of dedicated tsvector columns. This way
full-text search works the same way across all tables.

API changes:

* Changed /wiki_pages.json?search[body_matches] to match against only
  the body. Before `body_matches` matched against both the title and the body.

* Added /wiki_pages.json?search[title_or_body_matches] to match against
  both the title and the body.

* Fixed /dmails.json?search[message_matches] to match against both the
  title and body when doing a wildcard search. Before a wildcard search
  only matched against the body.

* Added /dmails.json?search[body_matches] to match against only the dmail body.

app/logical/concerns/searchable.rb

@@ -160,6 +160,16 @@ module Searchable
     where("#{qualified_column_for(attr)} ? :key", key: key)
   end
 
+  # https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-PARSING-DOCUMENTS
+  # https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES
+  def where_tsvector_matches(columns, query)
+    tsvectors = Array.wrap(columns).map do |column|
+      to_tsvector("pg_catalog.english", arel_table[column])
+    end.reduce(:concat)
+
+    where("(#{tsvectors.to_sql}) @@ plainto_tsquery('pg_catalog.english', :query)", query: query)
+  end
+
   def search_boolean_attribute(attr, params)
     if params[attr].present?
       boolean_attribute_matches(attr, params[attr])
@@ -194,18 +204,17 @@ module Searchable
     end
   end
 
-  def text_attribute_matches(attribute, value, index_column: nil)
-    return all unless value.present?
+  def text_attribute_matches(columns, query)
+    columns = Array.wrap(columns)
 
-    column = column_for_attribute(attribute)
-    qualified_column = "#{table_name}.#{column.name}"
-
-    if value =~ /\*/
-      where("lower(#{qualified_column}) LIKE :value ESCAPE E'\\\\'", value: value.mb_chars.downcase.to_escaped_for_sql_like)
-    elsif index_column.present?
-      where("#{table_name}.#{index_column} @@ plainto_tsquery('english', :value)", value: value)
+    if query.nil?
+      all
+    elsif query =~ /\*/
+      columns.map do |column|
+        where_ilike(column, query)
+      end.reduce(:or)
     else
-      where("to_tsvector('english', #{qualified_column}) @@ plainto_tsquery('english', :value)", value: value)
+      where_tsvector_matches(columns, query)
     end
   end
 
@@ -596,6 +605,20 @@ module Searchable
     end
   end
 
+  def sql_value(value)
+    if Arel.arel_node?(value)
+      value
+    elsif value.is_a?(String)
+      Arel::Nodes.build_quoted(value)
+    elsif value.is_a?(Symbol)
+      arel_table[value]
+    elsif value.is_a?(Array)
+      sql_array(value)
+    else
+      raise ArgumentError
+    end
+  end
+
   # Convert a Ruby array to an SQL array.
   #
   # @param values [Array]
@@ -603,4 +626,15 @@ module Searchable
   def sql_array(array)
     Arel.sql(ActiveRecord::Base.sanitize_sql(["ARRAY[?]", array]))
   end
+
+  # @example Tag.sql_function(:sum, Tag.arel_table[:post_count]).to_sql == "SUM(tags.post_count)"
+  def sql_function(name, *args)
+    Arel::Nodes::NamedFunction.new(name.to_s, args.map { |arg| sql_value(arg) })
+  end
+
+  # @example Note.to_tsvector("pg_catalog.english", :body).to_sql == "to_tsvector('pg_catalog.english', notes.body)"
+  # https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-PARSING-DOCUMENTS
+  def to_tsvector(config, column)
+    sql_function(:to_tsvector, config, column)
+  end
 end

app/models/comment.rb

@@ -28,7 +28,7 @@ class Comment < ApplicationRecord
   module SearchMethods
     def search(params)
       q = search_attributes(params, :id, :created_at, :updated_at, :is_deleted, :is_sticky, :do_not_bump_post, :body, :score, :post, :creator, :updater)
-      q = q.text_attribute_matches(:body, params[:body_matches], index_column: :body_index)
+      q = q.text_attribute_matches(:body, params[:body_matches])
 
       case params[:order]
       when "post_id", "post_id_desc"

app/models/dmail.rb

@@ -99,7 +99,8 @@ class Dmail < ApplicationRecord
     def search(params)
       q = search_attributes(params, :id, :created_at, :updated_at, :is_read, :is_deleted, :title, :body, :to, :from)
       q = q.text_attribute_matches(:title, params[:title_matches])
-      q = q.text_attribute_matches(:body, params[:message_matches], index_column: :message_index)
+      q = q.text_attribute_matches(:body, params[:body_matches])
+      q = q.text_attribute_matches([:title, :body], params[:message_matches])
 
       q = q.folder_matches(params[:folder])
 

app/models/forum_post.rb

@@ -53,7 +53,7 @@ class ForumPost < ApplicationRecord
 
     def search(params)
       q = search_attributes(params, :id, :created_at, :updated_at, :is_deleted, :body, :creator, :updater, :topic, :dtext_links, :votes, :tag_alias, :tag_implication, :bulk_update_request)
-      q = q.text_attribute_matches(:body, params[:body_matches], index_column: :text_index)
+      q = q.text_attribute_matches(:body, params[:body_matches])
 
       if params[:linked_to].present?
         q = q.wiki_link_matches(params[:linked_to])

app/models/forum_topic.rb

@@ -86,7 +86,7 @@ class ForumTopic < ApplicationRecord
 
     def search(params)
       q = search_attributes(params, :id, :created_at, :updated_at, :is_sticky, :is_locked, :is_deleted, :category_id, :title, :response_count, :creator, :updater, :forum_posts, :bulk_update_requests, :tag_aliases, :tag_implications)
-      q = q.text_attribute_matches(:title, params[:title_matches], index_column: :text_index)
+      q = q.text_attribute_matches(:title, params[:title_matches])
 
       if params[:is_private].to_s.truthy?
         q = q.private_only

app/models/note.rb

@@ -19,7 +19,7 @@ class Note < ApplicationRecord
   module SearchMethods
     def search(params)
       q = search_attributes(params, :id, :created_at, :updated_at, :is_active, :x, :y, :width, :height, :body, :version, :post)
-      q = q.text_attribute_matches(:body, params[:body_matches], index_column: :body_index)
+      q = q.text_attribute_matches(:body, params[:body_matches])
 
       q.apply_default_order(params)
     end

app/models/wiki_page.rb

@@ -70,7 +70,8 @@ class WikiPage < ApplicationRecord
 
     def search(params = {})
       q = search_attributes(params, :id, :created_at, :updated_at, :is_locked, :is_deleted, :body, :title, :other_names, :tag, :artist, :dtext_links)
-      q = q.text_attribute_matches(:body, params[:body_matches], index_column: :body_index)
+      q = q.text_attribute_matches(:body, params[:body_matches])
+      q = q.text_attribute_matches([:title, :body], params[:title_or_body_matches])
 
       if params[:title_normalize].present?
         q = q.where_like(:title, normalize_title(params[:title_normalize]))

app/views/wiki_pages/_search.html.erb

@@ -1,7 +1,7 @@
 <%= search_form_for(wiki_pages_path) do |f| %>
   <%= f.input :title_normalize, label: "Title", hint: "Use * for wildcard", input_html: { value: params[:search][:title_normalize], "data-autocomplete": "wiki-page" } %>
   <%= f.input :other_names_match, label: "Other names", hint: "Use * for wildcard", input_html: { value: params[:search][:other_names_match] } %>
-  <%= f.input :body_matches, label: "Body", hint: "Use * for wildcard", input_html: { value: params[:search][:body_matches] } %>
+  <%= f.input :title_or_body_matches, label: "Text", hint: "Use * for wildcard", input_html: { value: params[:search][:title_or_body_matches] } %>
   <%= f.input :linked_to, hint: "Find wikis linking to this wiki", input_html: { value: params[:search][:linked_to], "data-autocomplete": "wiki-page" } %>
   <%= f.input :is_deleted, label: "Deleted?", as: :select, include_blank: true, selected: params[:search][:is_deleted] %>
   <%= f.input :order, collection: [%w[Newest created_at], %w[Title title], %w[Posts post_count]], include_blank: true, selected: params[:search][:order] %>