diff --git a/app/logical/post_query_builder.rb b/app/logical/post_query_builder.rb index 72ffdf18e..71bfbcecf 100644 --- a/app/logical/post_query_builder.rb +++ b/app/logical/post_query_builder.rb @@ -514,6 +514,52 @@ class PostQueryBuilder relation end + def paginated_posts(page, small_search_threshold: Danbooru.config.small_search_threshold, **options) + posts = build.paginate(page, **options) + posts = optimize_search(posts, 30_000) + posts.load + end + + # XXX This is an ugly hack to try to deal with slow searches. By default, + # Postgres wants to do an index scan down the post id index for large + # order:id searches, and a bitmap scan on the tag index for small searches. + # The problem is that Postgres can't always tell whether a search is large or + # small. For large mutually-exclusive tags like 1girl + multiple_girls, + # Postgres assumes the search is large when actually it's small. For small + # tags, Postgres sometimes assumes tags in the 10k-50k range are large enough + # for a post id index scan, when in reality a tag index bitmap scan would be + # better. + def optimize_search(relation, small_search_threshold) + return relation unless small_search_threshold.present? + return relation unless relation.order_values == ["posts.id DESC"] + + if post_count.nil? + # If post_count is nil, then the search took too long to count and we don't + # know whether it's large or small. First we try it normally assuming it's + # large, then if that times out we try again assuming it's small. + posts = Post.with_timeout(1000) { relation.load } + posts = small_search(relation) if posts.nil? + elsif post_count <= small_search_threshold + # Otherwise if we know the search is small, then treat it as a small search. + posts = small_search(relation) + else + # Otherwise if we know it's large, treat it normally + posts = relation + end + + posts + end + + # Perform a search, forcing Postgres to do a bitmap scan on the tags index. + # https://www.postgresql.org/docs/current/runtime-config-query.html + def small_search(relation) + Post.transaction do + Post.connection.execute("SET LOCAL enable_seqscan = off") + Post.connection.execute("SET LOCAL enable_indexscan = off") + relation.load + end + end + def search_order(relation, order) case order.to_s.downcase when "id", "id_asc" @@ -891,6 +937,10 @@ class PostQueryBuilder end concerning :CountMethods do + def post_count + fast_count + end + # Return an estimate of the number of posts returned by the search. By # default, we try to use an estimated or cached count before doing an exact # count. @@ -1093,5 +1143,5 @@ class PostQueryBuilder end end - memoize :split_query + memoize :split_query, :post_count end diff --git a/app/logical/post_sets/post.rb b/app/logical/post_sets/post.rb index cca5a44fe..7e16df17b 100644 --- a/app/logical/post_sets/post.rb +++ b/app/logical/post_sets/post.rb @@ -7,7 +7,8 @@ module PostSets MAX_PER_PAGE = 200 MAX_SIDEBAR_TAGS = 25 - attr_reader :page, :random, :post_count, :format, :tag_string, :query, :normalized_query + attr_reader :page, :random, :format, :tag_string, :query, :normalized_query + delegate :post_count, to: :normalized_query def initialize(tags, page = 1, per_page = nil, user: CurrentUser.user, random: false, format: "html") @query = PostQueryBuilder.new(tags, user, tag_limit: user.tag_query_limit, safe_mode: CurrentUser.safe_mode?, hide_deleted_posts: user.hide_deleted_posts?) @@ -97,27 +98,16 @@ module PostSets random || query.find_metatag(:order) == "random" end - def get_post_count - if %w[json atom xml].include?(format.downcase) - # no need to get counts for formats that don't use a paginator - nil - else - normalized_query.fast_count - end - end - def get_random_posts ::Post.user_tag_match(tag_string).random(per_page) end def posts @posts ||= begin - @post_count = get_post_count - if is_random? get_random_posts.paginate(page, search_count: false, limit: per_page, max_limit: max_per_page).load else - normalized_query.build.paginate(page, count: post_count, search_count: !post_count.nil?, limit: per_page, max_limit: max_per_page).load + normalized_query.paginated_posts(page, count: post_count, search_count: !post_count.nil?, limit: per_page, max_limit: max_per_page).load end end end diff --git a/app/logical/related_tag_calculator.rb b/app/logical/related_tag_calculator.rb index cd9e73f6f..ad5a9d0fd 100644 --- a/app/logical/related_tag_calculator.rb +++ b/app/logical/related_tag_calculator.rb @@ -23,7 +23,7 @@ module RelatedTagCalculator # @param category [Integer] an optional tag category, to restrict the tags to a given category. # @return [Array] the set of similar tags, ordered by most similar def self.similar_tags_for_search(post_query, search_sample_size: 1000, tag_sample_size: 250, category: nil) - search_count = post_query.fast_count + search_count = post_query.post_count return [] if search_count.nil? search_sample_size = [search_count, search_sample_size].min diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 971cdcec8..cbbccebb0 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -318,6 +318,13 @@ module Danbooru [] end + # Tag searches with less than this many results will be considered "small + # searches" and optimized specially. This is unnecessary unless you have a + # Danbooru-sized database. + def small_search_threshold + nil + end + # Your Pixiv PHPSESSID cookie. Get this by logging in to Pixiv and using # the devtools to find the PHPSESSID cookie. This is need for Pixiv upload # support.