From fcc5521f63dec8f6eb2b2d18d7a7b725a6d8464f Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 29 Mar 2017 14:16:09 -0500 Subject: [PATCH 1/9] db: add updated_at index on wiki pages. The wiki pages sidebar is ordered by updated_at. The sidebar is present on all wiki pages, so this improves load times for all wikis. --- .../20170329185605_add_updated_at_index_on_wiki_pages.rb | 7 +++++++ db/structure.sql | 9 +++++++++ 2 files changed, 16 insertions(+) create mode 100644 db/migrate/20170329185605_add_updated_at_index_on_wiki_pages.rb diff --git a/db/migrate/20170329185605_add_updated_at_index_on_wiki_pages.rb b/db/migrate/20170329185605_add_updated_at_index_on_wiki_pages.rb new file mode 100644 index 000000000..0dfebb6bb --- /dev/null +++ b/db/migrate/20170329185605_add_updated_at_index_on_wiki_pages.rb @@ -0,0 +1,7 @@ +class AddUpdatedAtIndexOnWikiPages < ActiveRecord::Migration + def change + WikiPage.without_timeout do + add_index :wiki_pages, :updated_at + end + end +end diff --git a/db/structure.sql b/db/structure.sql index 7d30fe328..6789bdb88 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -7217,6 +7217,13 @@ CREATE UNIQUE INDEX index_wiki_pages_on_title ON wiki_pages USING btree (title); CREATE INDEX index_wiki_pages_on_title_pattern ON wiki_pages USING btree (title text_pattern_ops); +-- +-- Name: index_wiki_pages_on_updated_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_wiki_pages_on_updated_at ON wiki_pages USING btree (updated_at); + + -- -- Name: unique_schema_migrations; Type: INDEX; Schema: public; Owner: - -- @@ -7562,3 +7569,5 @@ INSERT INTO schema_migrations (version) VALUES ('20170316224630'); INSERT INTO schema_migrations (version) VALUES ('20170319000519'); +INSERT INTO schema_migrations (version) VALUES ('20170329185605'); + From 85a39cadf05cff3f4dfa58ba53b4f4158714851c Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 29 Mar 2017 13:46:27 -0500 Subject: [PATCH 2/9] /wiki_pages: avoid unnecessary COUNT(*). @wiki_pages.count issues a COUNT(*), which is unnecessary when we're just counting the ~20 @wiki_pages we've already loaded. --- app/controllers/wiki_pages_controller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/wiki_pages_controller.rb b/app/controllers/wiki_pages_controller.rb index 96b92d4ad..abcbe42c2 100644 --- a/app/controllers/wiki_pages_controller.rb +++ b/app/controllers/wiki_pages_controller.rb @@ -19,9 +19,9 @@ class WikiPagesController < ApplicationController respond_with(@wiki_pages) do |format| format.html do if params[:page].nil? || params[:page].to_i == 1 - if @wiki_pages.count == 1 + if @wiki_pages.length == 1 redirect_to(wiki_page_path(@wiki_pages.first)) - elsif @wiki_pages.count == 0 && params[:search][:title].present? && params[:search][:title] !~ /\*/ + elsif @wiki_pages.length == 0 && params[:search][:title].present? && params[:search][:title] !~ /\*/ redirect_to(wiki_pages_path(:search => {:title => "*#{params[:search][:title]}*"})) end end From bb2f0ff795b9707e9f0ec1c34183648f16afe1c4 Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 30 Mar 2017 13:17:49 -0500 Subject: [PATCH 3/9] /pools: make autocomplete use index. `name ilike ?` doesn't use the index. Use `lower(name) like ?` instead. --- app/models/pool.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/pool.rb b/app/models/pool.rb index 37932262d..0b8023979 100644 --- a/app/models/pool.rb +++ b/app/models/pool.rb @@ -45,9 +45,9 @@ class Pool < ActiveRecord::Base end def name_matches(name) - name = name.tr(" ", "_") + name = normalize_name_for_search(name) name = "*#{name}*" unless name =~ /\*/ - where("name ilike ? escape E'\\\\'", name.to_escaped_for_sql_like) + where("lower(name) like ? escape E'\\\\'", name.to_escaped_for_sql_like) end def search(params) @@ -139,6 +139,10 @@ class Pool < ActiveRecord::Base name.gsub(/\s+/, "_") end + def self.normalize_name_for_search(name) + normalize_name(name).mb_chars.downcase + end + def self.normalize_post_ids(post_ids, unique) hoge = post_ids.scan(/\d+/) if unique From 7f2859eb0085ef436c6b59cb575611837aeaa20a Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 30 Mar 2017 00:12:29 -0500 Subject: [PATCH 4/9] /pools: don't filter inactive pools from autocomplete. The inactive flag marks series pools that are finished (no more posts will be added), not pools that are deleted. Finished series should still be autocompleted. --- app/assets/javascripts/pools.js | 1 - 1 file changed, 1 deletion(-) diff --git a/app/assets/javascripts/pools.js b/app/assets/javascripts/pools.js index 8e78a8701..b5e1bf24b 100644 --- a/app/assets/javascripts/pools.js +++ b/app/assets/javascripts/pools.js @@ -27,7 +27,6 @@ $.ajax({ url: "/pools.json", data: { - "search[is_active]": "true", "search[name_matches]": req.term, "limit": 10 }, From 1a63fd6ac7ebc069da4823cce5bf8568ddbf32dc Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 30 Mar 2017 00:12:11 -0500 Subject: [PATCH 5/9] /artists: filter deleted artists from autocomplete. --- app/assets/javascripts/artists.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app/assets/javascripts/artists.js b/app/assets/javascripts/artists.js index d390f4998..d1800736a 100644 --- a/app/assets/javascripts/artists.js +++ b/app/assets/javascripts/artists.js @@ -54,6 +54,7 @@ url: "/artists.json", data: { "search[name]": "*" + req.term + "*", + "search[is_active]": true, "limit": 10 }, method: "get", From 674028588c406b26030bb6b7ae4e62e321bd0ec2 Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 30 Mar 2017 00:20:39 -0500 Subject: [PATCH 6/9] /artists: avoid COUNT(*), avoid N+1 queries on artist urls. * The /artists page issued a `SELECT count(*) FROM artists` because .paginate was missing search_count. * /artists.json had an N+1 problem on artist urls. This slowed down autocomplete. --- app/controllers/artists_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/artists_controller.rb b/app/controllers/artists_controller.rb index 7d6e67467..e38fff9ea 100644 --- a/app/controllers/artists_controller.rb +++ b/app/controllers/artists_controller.rb @@ -38,7 +38,7 @@ class ArtistsController < ApplicationController def index search_params = params[:search].present? ? params[:search] : params - @artists = Artist.search(search_params).order("id desc").paginate(params[:page], :limit => params[:limit]) + @artists = Artist.includes(:urls).search(search_params).order("id desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search]) respond_with(@artists) do |format| format.xml do render :xml => @artists.to_xml(:include => [:urls], :root => "artists") From fbba167f0c8a3f9c44ba70a2279f9072544e3237 Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 7 Apr 2017 16:53:27 -0500 Subject: [PATCH 7/9] artist.rb: fix ambiguous column references. Using `search[empty_only]=true` caused certain queries to throw an exception due to ambiguous column references after joining on the tags table. Example: https://danbooru.donmai.us/artists?search[empty_only]=true&search[name]=hammer* PG::AmbiguousColumn exception raised ERROR: column reference "name" is ambiguous LINE 1: ...ags"."name" = "artists"."name" WHERE (true) AND ((name LIKE ... ^ lib/danbooru/paginator/active_record_extension.rb:108:in `total_count' lib/danbooru/paginator/active_record_extension.rb:63:in `block in paginate_numbered' lib/danbooru/paginator/active_record_extension.rb:60:in `tap' lib/danbooru/paginator/active_record_extension.rb:60:in `paginate_numbered' lib/danbooru/paginator/active_record_extension.rb:15:in `paginate' app/controllers/artists_controller.rb:41:in `index' --- app/models/artist.rb | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/app/models/artist.rb b/app/models/artist.rb index 911e1b88f..accb734b1 100644 --- a/app/models/artist.rb +++ b/app/models/artist.rb @@ -21,6 +21,11 @@ class Artist < ActiveRecord::Base attr_accessible :is_active, :as => [:builder, :janitor, :moderator, :default, :admin] attr_accessible :is_banned, :as => :admin + scope :active, lambda { where(is_active: true) } + scope :deleted, lambda { where(is_active: false) } + scope :banned, lambda { where(is_banned: true) } + scope :unbanned, lambda { where(is_banned: false) } + module UrlMethods extend ActiveSupport::Concern @@ -328,14 +333,6 @@ class Artist < ActiveRecord::Base end module SearchMethods - def active - where("is_active = true") - end - - def banned - where("is_banned = true") - end - def url_matches(string) matches = find_all_by_url(string).map(&:id) @@ -362,33 +359,33 @@ class Artist < ActiveRecord::Base def other_names_match(string) if string =~ /\*/ && CurrentUser.is_builder? - where("other_names ILIKE ? ESCAPE E'\\\\'", string.to_escaped_for_sql_like) + where("artists.other_names ILIKE ? ESCAPE E'\\\\'", string.to_escaped_for_sql_like) else - where("other_names_index @@ to_tsquery('danbooru', E?)", Artist.normalize_name(string).to_escaped_for_tsquery) + where("artists.other_names_index @@ to_tsquery('danbooru', E?)", Artist.normalize_name(string).to_escaped_for_tsquery) end end def group_name_matches(name) stripped_name = normalize_name(name).to_escaped_for_sql_like - where("group_name LIKE ? ESCAPE E'\\\\'", stripped_name) + where("artists.group_name LIKE ? ESCAPE E'\\\\'", stripped_name) end def name_matches(name) stripped_name = normalize_name(name).to_escaped_for_sql_like - where("name LIKE ? ESCAPE E'\\\\'", stripped_name) + where("artists.name LIKE ? ESCAPE E'\\\\'", stripped_name) end def named(name) - where("name = ?", normalize_name(name)) + where(name: normalize_name(name)) end def any_name_matches(name) stripped_name = normalize_name(name).to_escaped_for_sql_like if name =~ /\*/ && CurrentUser.is_builder? - where("(name LIKE ? ESCAPE E'\\\\' OR other_names LIKE ? ESCAPE E'\\\\')", stripped_name, stripped_name) + where("(artists.name LIKE ? ESCAPE E'\\\\' OR artists.other_names LIKE ? ESCAPE E'\\\\')", stripped_name, stripped_name) else name_for_tsquery = normalize_name(name).to_escaped_for_tsquery - where("(name LIKE ? ESCAPE E'\\\\' OR other_names_index @@ to_tsquery('danbooru', E?))", stripped_name, name_for_tsquery) + where("(artists.name LIKE ? ESCAPE E'\\\\' OR artists.other_names_index @@ to_tsquery('danbooru', E?))", stripped_name, name_for_tsquery) end end @@ -413,7 +410,7 @@ class Artist < ActiveRecord::Base q = q.banned when /status:active/ - q = q.where("is_banned = false and is_active = true") + q = q.unbanned.active when /./ q = q.any_name_matches(params[:name]) @@ -422,23 +419,23 @@ class Artist < ActiveRecord::Base params[:order] ||= params.delete(:sort) case params[:order] when "name" - q = q.reorder("name") + q = q.reorder("artists.name") when "updated_at" - q = q.reorder("updated_at desc") + q = q.reorder("artists.updated_at desc") else - q = q.reorder("id desc") + q = q.reorder("artists.id desc") end if params[:is_active] == "true" q = q.active elsif params[:is_active] == "false" - q = q.where("is_active = false") + q = q.deleted end if params[:is_banned] == "true" q = q.banned elsif params[:is_banned] == "false" - q = q.where("is_banned = false") + q = q.unbanned end if params[:id].present? From 6b462c865e555d64bc1394ca6b803dc0af5aba30 Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 7 Apr 2017 17:20:57 -0500 Subject: [PATCH 8/9] /artists: sort autocomplete by post count. * Add search[order]=post_count param to /artists. * Make autocomplete do a prefix match ordered by post count, so that it works the same way that tag autocomplete does elsewhere. --- app/assets/javascripts/artists.js | 3 ++- app/controllers/artists_controller.rb | 2 +- app/models/artist.rb | 8 +++++--- app/views/artists/_search.html.erb | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/assets/javascripts/artists.js b/app/assets/javascripts/artists.js index d1800736a..2d89c5e21 100644 --- a/app/assets/javascripts/artists.js +++ b/app/assets/javascripts/artists.js @@ -53,8 +53,9 @@ $.ajax({ url: "/artists.json", data: { - "search[name]": "*" + req.term + "*", + "search[name]": req.term + "*", "search[is_active]": true, + "search[order]": "post_count", "limit": 10 }, method: "get", diff --git a/app/controllers/artists_controller.rb b/app/controllers/artists_controller.rb index e38fff9ea..d56df0b49 100644 --- a/app/controllers/artists_controller.rb +++ b/app/controllers/artists_controller.rb @@ -38,7 +38,7 @@ class ArtistsController < ApplicationController def index search_params = params[:search].present? ? params[:search] : params - @artists = Artist.includes(:urls).search(search_params).order("id desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search]) + @artists = Artist.includes(:urls).search(search_params).paginate(params[:page], :limit => params[:limit], :search_count => params[:search]) respond_with(@artists) do |format| format.xml do render :xml => @artists.to_xml(:include => [:urls], :root => "artists") diff --git a/app/models/artist.rb b/app/models/artist.rb index accb734b1..64d691771 100644 --- a/app/models/artist.rb +++ b/app/models/artist.rb @@ -419,11 +419,13 @@ class Artist < ActiveRecord::Base params[:order] ||= params.delete(:sort) case params[:order] when "name" - q = q.reorder("artists.name") + q = q.order("artists.name") when "updated_at" - q = q.reorder("artists.updated_at desc") + q = q.order("artists.updated_at desc") + when "post_count" + q = q.joins(:tag).order("tags.post_count desc") else - q = q.reorder("artists.id desc") + q = q.order("artists.id desc") end if params[:is_active] == "true" diff --git a/app/views/artists/_search.html.erb b/app/views/artists/_search.html.erb index 42a49edcc..3c256ff8d 100644 --- a/app/views/artists/_search.html.erb +++ b/app/views/artists/_search.html.erb @@ -15,7 +15,7 @@
- <%= select "search", "order", [["Recently created", "created_at"], ["Last updated", "updated_at"], ["Name", "name"]], :selected => params[:search][:order] %> + <%= select "search", "order", [["Recently created", "created_at"], ["Last updated", "updated_at"], ["Name", "name"], ["Post count", "post_count"]], :selected => params[:search][:order] %>
From b9693827c3e1d1998d306ffef6d2fe380e84e2b4 Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 30 Mar 2017 13:14:57 -0500 Subject: [PATCH 9/9] /wiki_pages: sort autocomplete by post count. * Add search[order]=post_count param to /wiki_pages. * Make autocomplete do a prefix match ordered by post count, so that it works the same way that tag autocomplete does elsewhere. --- app/assets/javascripts/wiki_pages.js | 3 ++- app/controllers/wiki_pages_controller.rb | 2 +- app/models/wiki_page.rb | 9 +++++++-- app/views/wiki_pages/search.html.erb | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/app/assets/javascripts/wiki_pages.js b/app/assets/javascripts/wiki_pages.js index 9446d4efd..81c81a1dc 100644 --- a/app/assets/javascripts/wiki_pages.js +++ b/app/assets/javascripts/wiki_pages.js @@ -20,8 +20,9 @@ $.ajax({ url: "/wiki_pages.json", data: { - "search[title]": "*" + req.term + "*", + "search[title]": req.term + "*", "search[hide_deleted]": "Yes", + "search[order]": "post_count", "limit": 10 }, method: "get", diff --git a/app/controllers/wiki_pages_controller.rb b/app/controllers/wiki_pages_controller.rb index abcbe42c2..213aefb29 100644 --- a/app/controllers/wiki_pages_controller.rb +++ b/app/controllers/wiki_pages_controller.rb @@ -15,7 +15,7 @@ class WikiPagesController < ApplicationController end def index - @wiki_pages = WikiPage.search(params[:search]).order("updated_at desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search]) + @wiki_pages = WikiPage.search(params[:search]).paginate(params[:page], :limit => params[:limit], :search_count => params[:search]) respond_with(@wiki_pages) do |format| format.html do if params[:page].nil? || params[:page].to_i == 1 diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index d31b7fb20..f2edc433e 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -79,10 +79,15 @@ class WikiPage < ActiveRecord::Base end params[:order] ||= params.delete(:sort) - if params[:order] == "time" || params[:order] == "Date" + case params[:order] + when "time" q = q.order("updated_at desc") - elsif params[:order] == "title" || params[:order] == "Name" + when "title" q = q.order("title") + when "post_count" + q = q.joins(:tag).order("tags.post_count desc") + else + q = q.order("updated_at desc") end q diff --git a/app/views/wiki_pages/search.html.erb b/app/views/wiki_pages/search.html.erb index fb312f803..9e7555e85 100644 --- a/app/views/wiki_pages/search.html.erb +++ b/app/views/wiki_pages/search.html.erb @@ -13,7 +13,7 @@
- <%= select "search", "order", ["Name", "Date"] %> + <%= select "search", "order", [%w[Name title], %w[Date time], %w[Posts post_count]] %>