jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

pagination: refactor to avoid counting pages in API.

Browse Source 
Previously the page-based (numbered) paginator would always count the
total_pages, even in API calls when it wasn't needed. This could be very
slow in some cases. Refactor so that total_pages isn't calculated unless
it's called.

While we're at it, refactor to condense all the sequential vs. numbered
pagination logic into one module. This incidentally fixes a couple more
bugs:

* "page=b0" returned all pages rather than nothing.
* Bad parameters like "page=blaha123" and "page=a123blah" were accepted.
This commit is contained in:
evazion
2019-10-07 20:41:41 -05:00
parent e1f37113b3
commit 93dd952949
12 changed files with 131 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
app/logical/danbooru/paginator/active_record_extension.rb
View File

@@ -1,119 +0,0 @@
require 'active_support/concern'
module Danbooru
module Paginator
module ActiveRecordExtension
extend ActiveSupport::Concern
module ClassMethods
def paginate(page, options = {})
@paginator_options = options
if use_sequential_paginator?(page)
paginate_sequential(page)
else
paginate_numbered(page)
end
end
def use_sequential_paginator?(page)
page =~ /[ab]\d+/i
end
def paginate_sequential(page)
if page =~ /b(\d+)/
paginate_sequential_before($1)
elsif page =~ /a(\d+)/
paginate_sequential_after($1)
else
paginate_sequential_before
end
end
def paginate_sequential_before(before_id = nil)
c = limit(records_per_page + 1)
if before_id.to_i > 0
c = c.where("#{table_name}.id < ?", before_id.to_i)
end
c = c.reorder("#{table_name}.id desc")
c = c.extending(SequentialCollectionExtension)
c.sequential_paginator_mode = :before
c
end
def paginate_sequential_after(after_id)
c = limit(records_per_page + 1).where("#{table_name}.id > ?", after_id.to_i).reorder("#{table_name}.id asc")
c = c.extending(SequentialCollectionExtension)
c.sequential_paginator_mode = :after
c
end
def paginate_numbered(page)
page = [page.to_i, 1].max
if page > Danbooru.config.max_numbered_pages
raise ::Danbooru::Paginator::PaginationError
end
extending(NumberedCollectionExtension).limit(records_per_page).offset((page - 1) * records_per_page).tap do |obj|
if records_per_page > 0
obj.total_pages = (obj.total_count.to_f / records_per_page).ceil
else
obj.total_pages = 1
end
obj.current_page = page
end
end
def records_per_page
option_for(:limit).to_i
end
# When paginating large tables, we want to avoid doing an expensive count query
# when the result won't even be used. So when calling paginate you can pass in
# an optional :search_count key which points to the search params. If these params
# exist, then assume we're doing a search and don't override the default count
# behavior. Otherwise, just return some large number so the paginator skips the
# count.
def option_for(key)
case key
when :limit
limit = @paginator_options.try(:[], :limit) || Danbooru.config.posts_per_page
if limit.to_i > 1_000
limit = 1_000
end
limit
when :count
if @paginator_options.has_key?(:search_count) && @paginator_options[:search_count].blank?
1_000_000
elsif @paginator_options[:count]
@paginator_options[:count]
else
nil
end
end
end
# taken from kaminari (https://github.com/amatsuda/kaminari)
def total_count
return option_for(:count) if option_for(:count)
c = except(:offset, :limit, :order)
c = c.reorder(nil)
c = c.count
c.respond_to?(:count) ? c.count : c
rescue ActiveRecord::StatementInvalid => e
if e.to_s =~ /statement timeout/
1_000_000
else
raise
end
end
end
end
end
end

15
app/logical/danbooru/paginator/numbered_collection_extension.rb
View File

@@ -1,15 +0,0 @@
module Danbooru
module Paginator
module NumberedCollectionExtension
attr_accessor :current_page, :total_pages
def is_first_page?
current_page == 1
end
def is_last_page?
current_page >= total_pages
end
end
end
end

6
app/logical/danbooru/paginator/pagination_error.rb
View File

@@ -1,6 +0,0 @@
module Danbooru
module Paginator
class PaginationError < Exception
end
end
end

34
app/logical/danbooru/paginator/sequential_collection_extension.rb
View File

@@ -1,34 +0,0 @@
module Danbooru
module Paginator
module SequentialCollectionExtension
attr_accessor :sequential_paginator_mode
def is_first_page?
if sequential_paginator_mode == :before
false
else
size <= records_per_page
end
end
def is_last_page?
if sequential_paginator_mode == :after
false
else
size <= records_per_page
end
end
# XXX Hack: in sequential pagination we fetch one more record than we need
# so that we can tell when we're on the first or last page. Here we override
# a rails internal method to discard that extra record. See #2044, #3642.
def records
if sequential_paginator_mode == :before
super.first(records_per_page)
else
super.first(records_per_page).reverse
end
end
end
end
end