jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

docs: add remaining docs for classes in app/logical.

Browse Source
This commit is contained in:
evazion
2021-06-23 20:32:59 -05:00
parent c6855261fe
commit 00ca7526bb
47 changed files with 705 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/jobs/application_job.rb
View File

@@ -1,3 +1,7 @@
# The base class for all background jobs on Danbooru.
#
# @see https://guides.rubyonrails.org/active_job_basics.html
# @see https://github.com/collectiveidea/delayed_job
class ApplicationJob < ActiveJob::Base
queue_as :default
queue_with_priority 0

1
app/jobs/delete_favorites_job.rb
View File

@@ -1,3 +1,4 @@
# A job that deletes a user's favorites when they delete their account.
class DeleteFavoritesJob < ApplicationJob
queue_as :default
queue_with_priority 20

2
app/jobs/delete_post_files_job.rb
View File

@@ -1,3 +1,5 @@
# A job that deletes a post's files after it's replaced, or a preprocessed
# upload is never completed.
class DeletePostFilesJob < ApplicationJob
queue_as :default
queue_with_priority 20

1
app/jobs/regenerate_post_job.rb
View File

@@ -1,3 +1,4 @@
# A job that regenerates a post's images and IQDB when a moderator requests it.
class RegeneratePostJob < ApplicationJob
queue_as :default
queue_with_priority 20

2
app/jobs/tag_batch_change_job.rb
View File

@@ -1,3 +1,5 @@
# A job that performs a mass update or tag nuke operation in a bulk update
# request.
class TagBatchChangeJob < ApplicationJob
queue_as :bulk_update

2
app/jobs/tag_rename_job.rb
View File

@@ -1,3 +1,5 @@
# A job that performs a tag rename or alias operation in a bulk update request.
# Jobs in the `bulk_update` queue are processed sequentially.
class TagRenameJob < ApplicationJob
queue_as :bulk_update

2
app/jobs/upload_preprocessor_delayed_start_job.rb
View File

@@ -1,3 +1,5 @@
# A job that downloads and generates thumbnails in the background for an image
# uploaded with the upload bookmarklet.
class UploadPreprocessorDelayedStartJob < ApplicationJob
queue_as :default
queue_with_priority(-1)

1
app/jobs/upload_service_delayed_start_job.rb
View File

@@ -1,3 +1,4 @@
# A job that tries to resume a preprocessed image upload.
class UploadServiceDelayedStartJob < ApplicationJob
queue_as :default
queue_with_priority(-1)

7
app/logical/concerns/deletable.rb
View File

@@ -1,3 +1,10 @@
# A concern that adds common helper methods to models that are soft deletable.
#
# @example
# class Post
# deletable
# end
#
module Deletable
extend ActiveSupport::Concern

40
app/logical/danbooru/http.rb
View File

@@ -9,6 +9,23 @@ require "danbooru/http/session"
require "danbooru/http/spoof_referrer"
require "danbooru/http/unpolish_cloudflare"
# The HTTP client used by Danbooru for all outgoing HTTP requests. A wrapper
# around the http.rb gem that adds some helper methods and custom behavior:
#
# * Redirects are automatically followed
# * Referers are automatically spoofed
# * Cookies are automatically remembered
# * Requests can be cached
# * Rate limited requests can be automatically retried
# * HTML and XML responses are automatically parsed
# * Sites using Cloudflare Polish are automatically bypassed
# * SSRF attempts are blocked
#
# @example
# http = Danbooru::Http.new
# response = http.get("https://danbooru.donmai.us/posts.json")
# json = response.parse
#
module Danbooru
class Http
class Error < StandardError; end
@@ -30,6 +47,7 @@ module Danbooru
.timeout(DEFAULT_TIMEOUT)
.headers("Accept-Encoding" => "gzip")
.headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
#.headers("User-Agent": Danbooru.config.canonical_app_name)
.use(:auto_inflate)
.use(redirector: { max_redirects: MAX_REDIRECTS })
.use(:session)
@@ -109,6 +127,13 @@ module Danbooru
end
concerning :DownloadMethods do
# Download a file from `url` and return a {MediaFile}.
#
# @param url [String] the URL to download
# @param file [Tempfile] the file to download the URL to
# @raise [DownloadError] if the server returns a non-200 OK response
# @raise [FileTooLargeError] if the file exceeds Danbooru's maximum download size.
# @return [Array<(HTTP::Response, MediaFile)>] the HTTP response and the downloaded file
def download_media(url, file: Tempfile.new("danbooru-download-", binmode: true))
response = get(url)
@@ -129,6 +154,13 @@ module Danbooru
protected
# Perform a HTTP request for the given URL. On error, return a fake 5xx
# response so the caller doesn't have to deal with exceptions.
#
# @param method [String] the HTTP method
# @param url [String] the URL to request
# @param options [Hash] the URL parameters
# @return [HTTP::Response] the HTTP response
def request(method, url, **options)
http.send(method, url, **options)
rescue OpenSSL::SSL::SSLError
@@ -145,6 +177,14 @@ module Danbooru
fake_response(599, "")
end
# Perform a HTTP request for the given URL, raising an error on 4xx or 5xx
# responses.
#
# @param method [String] the HTTP method
# @param url [String] the URL to request
# @param options [Hash] the URL parameters
# @raise [Danbooru::Http::Error] if the response was a 4xx or 5xx error
# @return [HTTP::Response] the HTTP response
def request!(method, url, **options)
response = request(method, url, **options)

10
app/logical/danbooru/http/retriable.rb
View File

@@ -1,9 +1,11 @@
# A HTTP::Feature that automatically retries requests that return a 429 error
# or a Retry-After header. Usage: `Danbooru::Http.use(:retriable).get(url)`.
# or a Retry-After header.
#
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
# @example
# Danbooru::Http.use(:retriable).get(url)
#
# @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
# @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
module Danbooru
class Http
class Retriable < HTTP::Feature

6
app/logical/danbooru/http/unpolish_cloudflare.rb
View File

@@ -1,5 +1,7 @@
# Bypass Cloudflare Polish (https://support.cloudflare.com/hc/en-us/articles/360000607372-Using-Cloudflare-Polish-to-compress-images)
# Detect sites using Cloudflare Polish and bypass it by adding a random
# cache-busting URL param.
#
# @see https://support.cloudflare.com/hc/en-us/articles/360000607372-Using-Cloudflare-Polish-to-compress-images
module Danbooru
class Http
class UnpolishCloudflare < HTTP::Feature

5
app/logical/danbooru/ip_address.rb
View File

@@ -1,7 +1,6 @@
# A wrapper around the IPAddress gem that adds some extra utility methods.
#
# https://github.com/ipaddress-gem/ipaddress
# @see https://github.com/ipaddress-gem/ipaddress
module Danbooru
class IpAddress
attr_reader :ip_address
@@ -26,7 +25,7 @@ module Danbooru
# If we're being reverse proxied behind Cloudflare, then Tor connections
# will appear to originate from 2405:8100:8000::/48.
# https://blog.cloudflare.com/cloudflare-onion-service/
# @see https://blog.cloudflare.com/cloudflare-onion-service/
def is_tor?
Danbooru::IpAddress.new("2405:8100:8000::/48").include?(ip_address)
end

2
app/logical/iqdb_client.rb
View File

@@ -64,7 +64,7 @@ class IqdbClient
post_ids = matches.map { |match| match["post_id"] }
posts = Post.where(id: post_ids).group_by(&:id).transform_values(&:first)
json.map do |match|
matches.map do |match|
post = posts.fetch(match["post_id"], nil)
match.with_indifferent_access.merge(post: post) if post
end.compact

45
app/logical/media_file.rb
View File

@@ -1,3 +1,9 @@
# A MediaFile represents an image, video, or flash file. It contains methods for
# detecting the file type, for generating a preview image, for getting metadata,
# and for resizing images.
#
# A MediaFile is a wrapper around a File object, and supports all methods
# supported by a File.
class MediaFile
extend Memoist
attr_accessor :file
@@ -5,6 +11,11 @@ class MediaFile
# delegate all File methods to `file`.
delegate *(File.instance_methods - MediaFile.instance_methods), to: :file
# Open a file or filename and return a MediaFile object.
#
# @param file [File, String] a filename or an open File object
# @param options [Hash] extra options for the MediaFile subclass.
# @return [MediaFile] the media file
def self.open(file, **options)
file = Kernel.open(file, "r", binmode: true) unless file.respond_to?(:read)
@@ -22,6 +33,9 @@ class MediaFile
end
end
# Detect a file's type based on the magic bytes in the header.
# @param [File] an open file
# @return [Symbol] the file's type
def self.file_ext(file)
header = file.pread(16, 0)
@@ -47,74 +61,105 @@ class MediaFile
:bin
end
# @return [Boolean] true if we can generate video previews.
def self.videos_enabled?
system("ffmpeg -version > /dev/null") && system("mkvmerge --version > /dev/null")
end
# Initialize a MediaFile from a regular File.
# @param file [File] the image file
def initialize(file, **options)
@file = file
end
# @return [Array<(Integer, Integer)>] the width and height of the file
def dimensions
[0, 0]
end
# @return [Integer] the width of the file
def width
dimensions.first
end
# @return [Integer] the height of the file
def height
dimensions.second
end
# @return [String] the MD5 hash of the file, as a hex string.
def md5
Digest::MD5.file(file.path).hexdigest
end
# @return [Symbol] the detected file extension
def file_ext
MediaFile.file_ext(file)
end
# @return [Integer] the size of the file in bytes
def file_size
file.size
end
# @return [Boolean] true if the file is an image
def is_image?
file_ext.in?([:jpg, :png, :gif])
end
# @return [Boolean] true if the file is a video
def is_video?
file_ext.in?([:webm, :mp4])
end
# @return [Boolean] true if the file is a Pixiv ugoira
def is_ugoira?
file_ext == :zip
end
# @return [Boolean] true if the file is a Flash file
def is_flash?
file_ext == :swf
end
# @return [Boolean] true if the file is corrupted in some way
def is_corrupt?
false
end
# @return [Boolean] true if the file is animated. Note that GIFs and PNGs may be animated.
def is_animated?
is_video?
end
# @return [Boolean] true if the file has an audio track. The track may not be audible.
def has_audio?
false
end
# @return [Float] the duration of the video or animation, in seconds.
def duration
0.0
end
# Return a preview of the file, sized to fit within the given width and
# height (preserving the aspect ratio).
#
# @param width [Integer] the max width of the image
# @param height [Integer] the max height of the image
# @param options [Hash] extra options when generating the preview
# @return [MediaFile] a preview file
def preview(width, height, **options)
nil
end
# Return a cropped preview version of the file, sized to fit exactly within
# the given width and height.
#
# @param width [Integer] the width of the cropped image
# @param height [Integer] the height of the cropped image
# @param options [Hash] extra options when generating the preview
# @return [MediaFile] a cropped preview file
def crop(width, height, **options)
nil
end

9
app/logical/media_file/image.rb
View File

@@ -1,3 +1,7 @@
# A MediaFile for a JPEG, PNG, or GIF file. Uses libvips for resizing images.
#
# @see https://github.com/libvips/ruby-vips
# @see https://libvips.github.io/libvips/API/current
class MediaFile::Image < MediaFile
# Taken from ArgyllCMS 2.0.0 (see also: https://ninedegreesbelow.com/photography/srgb-profile-comparison.html)
SRGB_PROFILE = "#{Rails.root}/config/sRGB.icm"
@@ -31,8 +35,8 @@ class MediaFile::Image < MediaFile
is_animated_gif? || is_animated_png?
end
# https://github.com/jcupitt/libvips/wiki/HOWTO----Image-shrinking
# http://jcupitt.github.io/libvips/API/current/Using-vipsthumbnail.md.html
# @see https://github.com/jcupitt/libvips/wiki/HOWTO----Image-shrinking
# @see http://jcupitt.github.io/libvips/API/current/Using-vipsthumbnail.md.html
def preview(width, height)
output_file = Tempfile.new(["image-preview", ".jpg"])
resized_image = image.thumbnail_image(width, height: height, **THUMBNAIL_OPTIONS)
@@ -62,6 +66,7 @@ class MediaFile::Image < MediaFile
file_ext == :png && APNGInspector.new(file.path).inspect!.animated?
end
# @return [Vips::Image] the Vips image object for the file
def image
Vips::Image.new_from_file(file.path, fail: true)
end

8
app/logical/media_file/ugoira.rb
View File

@@ -1,3 +1,10 @@
# A MediaFile for a Pixiv ugoira file.
#
# A Pixiv ugoira is an animation format that consists of a zip file containing
# JPEG or PNG images, one per frame, plus a JSON object containing the
# inter-frame delay timings. Each frame can have a different delay, therefore
# ugoiras can have a variable framerate. The frame data isn't stored inside the
# zip file, so it must be passed around separately.
class MediaFile::Ugoira < MediaFile
class Error < StandardError; end
attr_reader :frame_data
@@ -25,6 +32,7 @@ class MediaFile::Ugoira < MediaFile
preview_frame.crop(width, height)
end
# Convert a ugoira to a webm.
# XXX should take width and height and resize image
def convert
raise NotImplementedError, "can't convert ugoira to webm: ffmpeg or mkvmerge not installed" unless self.class.videos_enabled?

4
app/logical/media_file/video.rb
View File

@@ -1,3 +1,7 @@
# A MediaFile for a webm or mp4 video. Uses ffmpeg to generate preview
# thumbnails.
#
# @see https://github.com/streamio/streamio-ffmpeg
class MediaFile::Video < MediaFile
def dimensions
[video.width, video.height]

28
app/logical/pagination_extension.rb
View File

@@ -1,8 +1,36 @@
# A mixin that adds a `#paginate` method to an ActiveRecord relation.
#
# There are two pagination techniques. The first is page-based (numbered):
#
# https://danbooru.donmai.us/posts?page=1
# https://danbooru.donmai.us/posts?page=2
# https://danbooru.donmai.us/posts?page=3
#
# The second is id-based (sequential):
#
# https://danbooru.donmai.us/posts?page=a1000&limit=100
# https://danbooru.donmai.us/posts?page=a1100&limit=100
# https://danbooru.donmai.us/posts?page=a1200&limit=100
#
# https://danbooru.donmai.us/posts?page=b1000&limit=100
# https://danbooru.donmai.us/posts?page=b900&limit=100
# https://danbooru.donmai.us/posts?page=b800&limit=100
#
# where a1000 means "after id 1000" and b1000 means "before id 1000".
#
module PaginationExtension
class PaginationError < StandardError; end
attr_accessor :current_page, :records_per_page, :paginator_count, :paginator_mode, :paginator_page_limit
# Paginate an ActiveRecord relation. Returns a relation for the given page and number of posts per page.
#
# @param page [String] the page number, or an "aNNN" or "bNNN" string
# @param limit [Integer] the number of posts per page
# @param max_limit [Integer] the maximum number of posts per page the user can view
# @param page_limit [Integer] the highest page the user can view
# @param count [Integer] the precalculated number of search results, or nil to calculate it
# @param search_count [Object] if truthy, don't calculate the number of results; assume a large number of results
def paginate(page, limit: nil, max_limit: 1000, page_limit: CurrentUser.user.page_limit, count: nil, search_count: nil)
@records_per_page = limit || Danbooru.config.posts_per_page
@records_per_page = @records_per_page.to_i.clamp(1, max_limit)

66
app/logical/post_query_builder.rb
View File

@@ -1,8 +1,16 @@
require "strscan"
# A PostQueryBuilder represents a post search. It contains all logic for parsing
# and executing searches.
#
# @example
# PostQueryBuilder.new("touhou rating:s").build
# #=> <set of posts>
#
class PostQueryBuilder
extend Memoist
# Raised when the number of tags exceeds the user's tag limit.
class TagLimitError < StandardError; end
# How many tags a `blah*` search should match.
@@ -58,12 +66,19 @@ class PostQueryBuilder
COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } +
CATEGORY_COUNT_METATAGS.flat_map { |str| [str, "#{str}_asc"] }
# Tags that don't count against the user's tag limit.
UNLIMITED_METATAGS = %w[status rating limit]
attr_reader :query_string, :current_user, :tag_limit, :safe_mode, :hide_deleted_posts
alias_method :safe_mode?, :safe_mode
alias_method :hide_deleted_posts?, :hide_deleted_posts
# Initialize a post query.
# @param query_string [String] the tag search
# @param current_user [User] the user performing the search
# @param tag_limit [Integer] the user's tag limit
# @param safe_mode [Boolean] whether safe mode is enabled. if true, return only rating:s posts.
# @param hide_deleted_posts [Boolean] if true, filter out status:deleted posts.
def initialize(query_string, current_user = User.anonymous, tag_limit: nil, safe_mode: false, hide_deleted_posts: false)
@query_string = query_string
@current_user = current_user
@@ -638,6 +653,7 @@ class PostQueryBuilder
relation.find_ordered(ids)
end
# @raise [TagLimitError] if the number of tags exceeds the user's tag limit
def validate!
tag_count = terms.count { |term| !is_unlimited_tag?(term) }
@@ -646,11 +662,14 @@ class PostQueryBuilder
end
end
# @return [Boolean] true if the metatag doesn't count against the user's tag limit
def is_unlimited_tag?(term)
term.type == :metatag && term.name.in?(UNLIMITED_METATAGS)
end
concerning :ParseMethods do
# Parse the search into a list of search terms. A search term is a tag or a metatag.
# @return [Array<OpenStruct>] a list of terms
def scan_query
terms = []
query = query_string.to_s.gsub(/[[:space:]]/, " ")
@@ -686,6 +705,9 @@ class PostQueryBuilder
terms
end
# Parse a single-quoted, double-quoted, or unquoted string. Used for parsing metatag values.
# @param scanner [StringScanner] the current parser state
# @return [Array<(String, Boolean)>] the string and whether it was quoted
def scan_string(scanner)
if scanner.scan(/"((?:\\"|[^"])*)"/)
value = scanner.captures.first.gsub(/\\(.)/) { $1 }
@@ -702,6 +724,9 @@ class PostQueryBuilder
[value, quoted]
end
# Split the search query into a list of strings, one per search term.
# Roughly the same as splitting on spaces, but accounts for quoted strings.
# @return [Array<String>] the list of terms
def split_query
terms.map do |term|
type, name, value = term.type, term.name, term.value
@@ -724,10 +749,16 @@ class PostQueryBuilder
end
end
# Parse a tag edit string into a list of strings, one per search term.
# @return [Array<String>] the list of terms
def parse_tag_edit
split_query
end
# Parse a simple string value into a Ruby type.
# @param object [String] the value to parse
# @param type [Symbol] the value's type
# @return [Object] the parsed value
def parse_cast(object, type)
case type
when :enum
@@ -790,6 +821,9 @@ class PostQueryBuilder
end
end
# Parse a metatag range value of the given type. For example: 1..10.
# @param string [String] the metatag value
# @param type [Symbol] the value's type
def parse_range(string, type)
range = case string
when /\A(.+?)\.\.\.(.+)/ # A...B
@@ -846,6 +880,14 @@ class PostQueryBuilder
end
concerning :CountMethods do
# Return an estimate of the number of posts returned by the search. By
# default, we try to use an estimated or cached count before doing an exact
# count.
#
# @param timeout [Integer] the database timeout
# @param estimate_count [Boolean] if true, estimate the count with inexact methods
# @param skip_cache [Boolean] if true, don't use the cached count
# @return [Integer, nil] the number of posts, or nil on timeout
def fast_count(timeout: 1_000, estimate_count: true, skip_cache: false)
count = nil
count = estimated_count if estimate_count
@@ -864,6 +906,7 @@ class PostQueryBuilder
end
end
# Estimate the count by parsing the Postgres EXPLAIN output.
def estimated_row_count
ExplainParser.new(build).row_count
end
@@ -896,6 +939,8 @@ class PostQueryBuilder
end
end
# @return [Boolean] true if the search depends on the current user because
# of permissions or privacy settings.
def is_user_dependent_search?
metatags.any? do |metatag|
metatag.name.in?(%w[upvoter upvote downvoter downvote search flagger fav ordfav favgroup ordfavgroup]) ||
@@ -906,6 +951,8 @@ class PostQueryBuilder
end
concerning :NormalizationMethods do
# Normalize a search by sorting tags and applying aliases.
# @return [PostQueryBuilder] the normalized query
def normalized_query(implicit: true, sort: true)
post_query = dup
post_query.terms.concat(implicit_metatags) if implicit
@@ -914,6 +961,7 @@ class PostQueryBuilder
post_query
end
# Apply aliases to all tags in the query.
def normalize_aliases!
tag_names = tags.map(&:name)
tag_aliases = tag_names.zip(TagAlias.to_aliased(tag_names)).to_h
@@ -924,10 +972,14 @@ class PostQueryBuilder
end
end
# Normalize the tag order.
def normalize_order!
terms.sort_by!(&:to_s).uniq!
end
# Implicit metatags are metatags added by the user's account settings.
# rating:s is implicit under safe mode. -status:deleted is implicit when the
# "hide deleted posts" setting is on.
def implicit_metatags
metatags = []
metatags << OpenStruct.new(type: :metatag, name: "rating", value: "s") if safe_mode?
@@ -947,34 +999,42 @@ class PostQueryBuilder
split_query.join(" ")
end
# The list of search terms. This includes regular tags and metatags.
def terms
@terms ||= scan_query
end
# The list of regular tags in the search.
def tags
terms.select { |term| term.type == :tag }
end
# The list of metatags in the search.
def metatags
terms.select { |term| term.type == :metatag }
end
# Find all metatags with the given names.
def select_metatags(*names)
metatags.select { |term| term.name.in?(names.map(&:to_s)) }
end
# Find the first metatag with any of the given names.
def find_metatag(*metatags)
select_metatags(*metatags).first.try(:value)
end
# @return [Boolean] true if the search has a metatag with any of the given names.
def has_metatag?(*metatag_names)
metatags.any? { |term| term.name.in?(metatag_names.map(&:to_s).map(&:downcase)) }
end
# @return [Boolean] true if the search has a single regular tag, with any number of metatags.
def has_single_tag?
tags.size == 1 && !tags.first.wildcard
end
# @return [Boolean] true if the search is a single metatag search for the given metatag.
def is_metatag?(name, value = nil)
if value.nil?
is_single_term? && has_metatag?(name)
@@ -983,27 +1043,33 @@ class PostQueryBuilder
end
end
# @return [Boolean] true if the search doesn't have any tags or metatags.
def is_empty_search?
terms.size == 0
end
# @return [Boolean] true if the search consists of a single tag or metatag.
def is_single_term?
terms.size == 1
end
# @return [Boolean] true if the search has a single tag, possibly with wildcards or negation.
def is_single_tag?
is_single_term? && tags.size == 1
end
# @return [Boolean] true if the search has a single tag, without any wildcards or operators.
def is_simple_tag?
tag = tags.first
is_single_tag? && !tag.negated && !tag.optional && !tag.wildcard
end
# @return [Boolean] true if the search has a single tag with a wildcard
def is_wildcard_search?
is_single_tag? && tags.first.wildcard
end
# @return [Tag, nil] the tag if the search is for a simple tag, otherwise nil
def simple_tag
return nil if !is_simple_tag?
Tag.find_by_name(tags.first.name)

5
app/logical/post_search_context.rb
View File

@@ -1,3 +1,8 @@
# A PostSearchContext handles navigating to the next or previous post in a
# search. This is used in the search navbar above or below posts.
#
# @see PostNavbarComponent
# @see PostsController#show_seq
class PostSearchContext
extend Memoist
attr_reader :id, :seq, :tags

4
app/logical/post_sets/post.rb
View File

@@ -1,3 +1,7 @@
# A PostSet is a set of posts returned by a search. This contains helper
# methods used on the post index page.
#
# @see PostsController#index
module PostSets
class Post
MAX_PER_PAGE = 200

22
app/logical/rate_limiter.rb
View File

@@ -1,3 +1,13 @@
# A RateLimiter handles HTTP rate limits for controller actions. Rate limits are
# based on the user, the user's IP, and the controller action.
#
# A RateLimiter is backed by RateLimit objects stored in the database, which
# track the rate limits with a token bucket algorithm. A RateLimiter object
# usually has two RateLimits, one for the current user and one for their IP.
#
# @see RateLimit
# @see ApplicationController#check_rate_limit
# @see https://en.wikipedia.org/wiki/Token_bucket
class RateLimiter
class RateLimitError < StandardError; end
@@ -11,6 +21,15 @@ class RateLimiter
@burst = burst
end
# Create a RateLimiter object for the current controller, action, user, and
# IP. A RateLimiter usually has two RateLimits, one for the user and one for
# their IP. The action is limited if either the user or their IP are limited.
#
# @param controller_name [String] the current controller
# @param action_name [String] the current controller action
# @param user [User] the current user
# @param ip_addr [String] the user's IP address
# @return [RateLimit] the rate limit for the action
def self.for_action(controller_name, action_name, user, ip_addr)
action = "#{controller_name}:#{action_name}"
keys = [(user.cache_key unless user.is_anonymous?), "ip/#{ip_addr.to_s}"].compact
@@ -33,10 +52,12 @@ class RateLimiter
RateLimiter.new(action, keys, rate: rate, burst: burst)
end
# @raise [RateLimitError] if the action is limited
def limit!
raise RateLimitError if limited?
end
# @return [Boolean] true if the action is limited for the user or their IP
def limited?
rate_limits.any?(&:limited?)
end
@@ -46,6 +67,7 @@ class RateLimiter
super(options).except("keys", "rate_limits").merge(limits: hash)
end
# Update or create the rate limits associated with this action.
def rate_limits
@rate_limits ||= RateLimit.create_or_update!(action: action, keys: keys, cost: cost, rate: rate, burst: burst)
end

28
app/logical/recommender_service.rb
View File

@@ -1,3 +1,14 @@
# An API client for the post recommendation service. The recommendation service
# is a separate Python microservice that generates recommended posts for users
# and posts. This client merely fetches the pre-generated recommendations from
# the service.
#
# Recommendations are generated based on user favorites using the Python
# `implicit` library.
#
# @see https://github.com/evazion/recommender
# @see https://github.com/benfred/implicit
# @see RecommendedPostsController
module RecommenderService
module_function
@@ -9,14 +20,23 @@ module RecommenderService
Danbooru.config.recommender_server.present?
end
# @return [Boolean] True if the post has recommendations. Posts without enough
# favorites aren't generated recommendations.
def available_for_post?(post)
enabled? && post.fav_count > MIN_POST_FAVS
end
# @return [Boolean] True if the user has recommendations. Users without enough
# favorites aren't generated recommendations.
def available_for_user?(user)
enabled? && user.favorite_count > MIN_USER_FAVS
end
# Return a set of recommended posts for a user.
# @param user [User] the user to get recommendations for
# @param tags [String] a tag search to filter recommendations by
# @param limit [Integer] the maximum number of recommendations to get
# @return [Hash] The recommended posts. A hash with `score` and `post` keys.
def recommend_for_user(user, tags: nil, limit: 50)
response = Danbooru::Http.cache(CACHE_LIFETIME).get("#{Danbooru.config.recommender_server}/recommend/#{user.id}", params: { limit: limit })
return [] if response.status != 200
@@ -24,6 +44,11 @@ module RecommenderService
process_recs(response.parse, tags: tags, uploader: user, favoriter: user)
end
# Return a set of recommended posts for a post.
# @param post [Post] the post to get recommendations for
# @param tags [String] a tag search to filter recommendations by
# @param limit [Integer] the maximum number of recommendations to get
# @return [Hash] The recommended posts. A hash with `score` and `post` keys.
def recommend_for_post(post, tags: nil, limit: 50)
response = Danbooru::Http.cache(CACHE_LIFETIME).get("#{Danbooru.config.recommender_server}/similar/#{post.id}", params: { limit: limit })
return [] if response.status != 200
@@ -31,6 +56,8 @@ module RecommenderService
process_recs(response.parse, post: post, tags: tags)
end
# Process a set of recommendations to filter out posts the user uploaded
# themselves, or has already favorited, or that don't match a tag search.
def process_recs(recs, post: nil, uploader: nil, favoriter: nil, tags: nil)
posts = Post.where(id: recs.map(&:first))
posts = posts.where.not(id: post.id) if post
@@ -44,6 +71,7 @@ module RecommenderService
recs
end
# Handle the RecommendedPostsController#index method.
def search(params)
if params[:user_name].present?
user = User.find_by_name(params[:user_name])

46
app/logical/related_tag_calculator.rb
View File

@@ -1,4 +1,27 @@
# Calculate the tags similar to a given tag. Two tags are similar if they have
# nearly the same set of posts, and nearly the same size.
#
# Similarity is calculated using cosine similarity, which is defined as the
# number of items two sets A and B have in common, divided by sqrt(||A|| * ||B||),
# where ||A|| is the size of A. The sqrt of the sizes can be thought of as a
# normalizing factor, to normalize the number of posts in common to a 0.0 - 1.0
# range.
#
# We optimize the calculation by sampling only 1000 random posts from the tag,
# calculating the number of times each tag on those posts appears, then taking
# the top 250 most frequently appearing tags as candidates for similar tags.
#
# Related tags are used for the tag sidebar when doing a search, and for the
# related tags feature when tagging a post.
#
# @see https://en.wikipedia.org/wiki/Cosine_similarity
module RelatedTagCalculator
# Return the set of tags similar to the given search.
# @param post_query [PostQueryBuilder] the search to find similar tags for.
# @param search_sample_size [Integer] the number of posts to sample from the search
# @param tag_sample_size [Integer] the number of tags to calculate similarity for
# @param category [Integer] an optional tag category, to restrict the tags to a given category.
# @return [Array<Tag>] the set of similar tags, ordered by most similar
def self.similar_tags_for_search(post_query, search_sample_size: 1000, tag_sample_size: 250, category: nil)
search_count = post_query.fast_count
return [] if search_count.nil?
@@ -15,11 +38,20 @@ module RelatedTagCalculator
tags
end
# Return the set of tags most frequently appearing in the given search.
# @param post_query [PostQueryBuilder] the search to find frequent tags for.
# @param search_sample_size [Integer] the number of posts to sample from the search
# @param category [Integer] an optional tag category, to restrict the tags to a given category.
# @return [Array<Tag>] the set of frequent tags, ordered by most frequent
def self.frequent_tags_for_search(post_query, search_sample_size: 1000, category: nil)
sample_posts = post_query.build.reorder(:md5).limit(search_sample_size)
frequent_tags_for_post_relation(sample_posts, category: category)
end
# Return the set of tags most frequently appearing in the given set of posts.
# @param posts [ActiveRecord::Relation<Post>] the set of posts
# @param category [Integer] an optional tag category, to restrict the tags to a given category.
# @return [Array<Tag>] the set of frequent tags, ordered by most frequent
def self.frequent_tags_for_post_relation(posts, category: nil)
tag_counts = Post.from(posts).with_unflattened_tags.group("tag").select("tag, COUNT(*) AS overlap_count")
@@ -31,11 +63,20 @@ module RelatedTagCalculator
tags
end
# Return the set of tags most frequently appearing in the given array of posts.
# @param posts [Array<Post>] the array of posts
# @return [Array<Tag>] the set of frequent tags, ordered by most frequent
def self.frequent_tags_for_post_array(posts)
tags_with_counts = posts.flat_map(&:tag_array).group_by(&:itself).transform_values(&:size)
tags_with_counts.sort_by { |tag_name, count| [-count, tag_name] }.map(&:first)
end
# Return a cached set of tags similar to the given search.
# @param post_query [PostQueryBuilder] the search to find similar tags for.
# @param max_tags [Integer] the maximum number of tags to return
# @param search_timeout [Integer] the database timeout for the search
# @param cache_timeout [Integer] the length of time to cache the results
# @return [Array<String>] the set of similar tag names, ordered by most similar
def self.cached_similar_tags_for_search(post_query, max_tags, search_timeout: 2000, cache_timeout: 8.hours)
Cache.get(cache_key(post_query), cache_timeout, race_condition_ttl: 60.seconds) do
ApplicationRecord.with_timeout(search_timeout, []) do
@@ -44,6 +85,11 @@ module RelatedTagCalculator
end
end
# Return a cache key for the given search. Some searches are cached on a
# per-user basis because they depend on the current user (for example,
# searches for private favorites, favgroups, or saved searches).
# @param post_query [PostQueryBuilder] the post search
# @return [String] the cache key
def self.cache_key(post_query)
if post_query.is_user_dependent_search?
"similar_tags[#{post_query.current_user.id}]:#{post_query.to_s}"

2
app/logical/related_tag_query.rb
View File

@@ -1,3 +1,5 @@
# Handle finding related tags by the {RelatedTagsController}. Used for finding
# related tags when tagging a post.
class RelatedTagQuery
include ActiveModel::Serializers::JSON
include ActiveModel::Serializers::Xml

20
app/logical/reportbooru_service.rb
View File

@@ -1,3 +1,7 @@
# An API client for the Reportbooru service. Reportbooru tracks post view
# counts, post search counts, and missed search counts.
#
# @see https://github.com/r888888888/reportbooru
class ReportbooruService
attr_reader :http, :reportbooru_server
@@ -6,10 +10,14 @@ class ReportbooruService
@http = http.timeout(1)
end
# @return [Boolean] true if Reportbooru is configured
def enabled?
reportbooru_server.present?
end
# Get the list of today's top missed searches.
# @param expires_in [Integer] the length of time to cache the results
# @return [Hash<String, Integer>] a map from searches to search counts
def missed_search_rankings(expires_in: 1.minute)
return [] unless enabled?
@@ -20,10 +28,18 @@ class ReportbooruService
body.lines.map(&:split).map { [_1, _2.to_i] }
end
# Get the list of the day's top searches.
# @param date [Date] the date to check (YYYY-MM-DD)
# @param expires_in [Integer] the length of time to cache the results
# @return [Array<Array<(String, Float)>>] a map from searches to search counts
def post_search_rankings(date, expires_in: 1.minute)
request("#{reportbooru_server}/post_searches/rank?date=#{date}", expires_in)
end
# Get the list of the day's most viewed posts.
# @param date [Date] the date to check (YYYY-MM-DD)
# @param expires_in [Integer] the length of time to cache the results
# @return [Array<Array<(String, Float)>>] a map from post ids to view counts
def post_view_rankings(date, expires_in: 1.minute)
request("#{reportbooru_server}/post_views/rank?date=#{date}", expires_in)
end
@@ -40,6 +56,10 @@ class ReportbooruService
ranking.take(limit).map { |x| Post.find(x[0]) }
end
# Send a request to Reportbooru
# @param url [String] the full Reportbooru URL
# @param expires_in [Integer] the length of time to cache the results
# @return [Object] the parsed JSON response
def request(url, expires_in)
return [] unless enabled?

9
app/logical/routes.rb
View File

@@ -1,6 +1,11 @@
# Allow Rails URL helpers to be used outside of views.
# Example: Routes.posts_path(tags: "touhou") => /posts?tags=touhou
#
# @example
# Routes.posts_path(tags: "touhou")
# => "/posts?tags=touhou"
#
# @see config/routes.rb
# @see https://guides.rubyonrails.org/routing.html
class Routes
include Singleton
include Rails.application.routes.url_helpers

5
app/logical/server_status.rb
View File

@@ -1,3 +1,8 @@
# Returns status information about the running server, including software
# versions, basic load info, and Redis and Postgres info.
#
# @see StatusController
# @see https://danbooru.donmai.us/status
class ServerStatus
extend Memoist
include ActiveModel::Serializers::JSON

37
app/logical/session_loader.rb
View File

@@ -1,14 +1,27 @@
# Loads the current user from their session cookies or API key. Used by the
# ApplicationController to set the CurrentUser global early during the HTTP
# request cycle.
#
# @see ApplicationController#set_current_user
# @see CurrentUser
class SessionLoader
class AuthenticationFailure < StandardError; end
attr_reader :session, :request, :params
# Initialize the session loader.
# @param request the HTTP request
def initialize(request)
@request = request
@session = request.session
@params = request.parameters
end
# Attempt to log a user in with the given username and password. Records a
# login attempt event and returns the user if successful.
# @param name [String] the username
# @param password [String] the user's password
# @return [User, nil] the user if the password was correct, otherwise nil
def login(name, password)
user = User.find_by_name(name)
@@ -30,6 +43,7 @@ class SessionLoader
end
end
# Logs the current user out. Deletes their session cookie and records a logout event.
def logout
session.delete(:user_id)
session.delete(:last_authenticated_at)
@@ -37,6 +51,17 @@ class SessionLoader
UserEvent.create_from_request!(CurrentUser.user, :logout, request)
end
# Sets the current user. Runs on each HTTP request. The user is set based on
# their API key, their session cookie, or the signed user id param (used when
# reseting a password from an magic email link)
#
# Also performs post-load actions, including updating the user's last login
# timestamp, their last used IP, their timezone, their database timeout, their
# country, whether safe mode is enabled, their session cookie, and unbanning
# banned users if their ban is expired.
#
# @see ApplicationController#set_current_user
# @see CurrentUser
def load
CurrentUser.user = User.anonymous
CurrentUser.ip_addr = request.remote_ip
@@ -61,6 +86,7 @@ class SessionLoader
DanbooruLogger.add_session_attributes(request, session, CurrentUser.user)
end
# @return [Boolean] true if the current request has an API key
def has_api_authentication?
request.authorization.present? || params[:login].present? || (params[:api_key].present? && params[:api_key].is_a?(String))
end
@@ -72,6 +98,8 @@ class SessionLoader
ActiveRecord::Base.connection.execute("set statement_timeout = #{timeout}")
end
# Sets the current API user based on either the `login` + `api_key` URL params,
# or HTTP Basic Auth.
def load_session_for_api
if request.authorization
authenticate_basic_auth
@@ -82,6 +110,7 @@ class SessionLoader
end
end
# Sets the current API user based on the HTTP Basic Auth params.
def authenticate_basic_auth
credentials = ::Base64.decode64(request.authorization.split(' ', 2).last || '')
login, api_key = credentials.split(/:/, 2)
@@ -89,6 +118,12 @@ class SessionLoader
authenticate_api_key(login, api_key)
end
# Sets the current user if their API key is valid.
# @param name [String] the user name
# @param key [String] the API key
# @raise AuthenticationFailure if the API key is invalid
# @raise User::PrivilegeError if the API key doesn't have the required
# permissions for this endpoint
def authenticate_api_key(name, key)
user, api_key = User.find_by_name(name)&.authenticate_api_key(key)
raise AuthenticationFailure if user.blank?
@@ -97,12 +132,14 @@ class SessionLoader
CurrentUser.user = user
end
# Set the current user based on the `signed_user_id` URL param. This param is used by the reset password email.
# XXX use rails 6.1 signed ids (https://github.com/rails/rails/blob/6-1-stable/activerecord/CHANGELOG.md)
def load_param_user(signed_user_id)
session[:user_id] = Danbooru::MessageVerifier.new(:login).verify(signed_user_id)
load_session_user
end
# Set the current user based on the `user_id` session cookie.
def load_session_user
user = User.find_by_id(session[:user_id])
CurrentUser.user = user if user

15
app/logical/set_diff.rb
View File

@@ -1,6 +1,12 @@
# Calculate a diff between two sets of strings. Used to calculate diffs between
# artist URLs and between other names. Tries to compare each set to detect which
# strings were added, which were removed, and which were changed.
class SetDiff
attr_reader :additions, :removals, :added, :removed, :changed, :unchanged
# Initialize a diff between two sets of strings.
# @param this_list [Array<String>] the new list of strings
# @param other_list [Array<String>] the old list of strings
def initialize(this_list, other_list)
this, other = this_list.to_a, other_list.to_a
@@ -10,6 +16,12 @@ class SetDiff
@added, @removed, @changed = changes(additions, removals)
end
# Returns the strings that were either completely newly added, completely
# removed, or simply updated.
# @param added [Array<String>] the strings that were added to this_list
# @param removed [Array<String>] the strings that were removed from other_list
# @return [Array<(Array<String>, Array<String>, Array<String>)>] the list of
# additions, removals, and changed strings.
def changes(added, removed)
changed = []
@@ -24,6 +36,9 @@ class SetDiff
[added, removed, changed]
end
# Finds the strings most similar to the `candidate` string among the `candidates` set.
# @param string [String] the string to find closest matches with in the `candidates` set.
# @param candidates [Array<String>] the set of candidate strings
def find_similar(string, candidates, max_dissimilarity: 0.70)
distance = ->(other) { ::DidYouMean::Levenshtein.distance(string, other) }
max_distance = string.size * max_dissimilarity

28
app/logical/spam_detector.rb
View File

@@ -1,10 +1,13 @@
# https://github.com/joshfrench/rakismet
# https://akismet.com/development/api/#comment-check
# Detects whether a dmail, comment, or forum post seems like spam. Autobans
# users who receive more than 10 spam reports in an hour. Uses the Akismet spam
# detection service.
#
# @see https://github.com/joshfrench/rakismet
# @see https://akismet.com/development/api/#comment-check
class SpamDetector
include Rakismet::Model
# if a person receives more than 10 automatic spam reports within a 1 hour
# If a person receives more than 10 automatic spam reports within a 1 hour
# window, automatically ban them forever.
AUTOBAN_THRESHOLD = 10
AUTOBAN_WINDOW = 1.hour
@@ -12,6 +15,7 @@ class SpamDetector
attr_accessor :record, :user, :user_ip, :content, :comment_type
# The attributes to pass to Akismet
rakismet_attrs author: proc { user.name },
author_email: proc { user.email_address&.address },
blog_lang: "en",
@@ -20,17 +24,23 @@ class SpamDetector
content: :content,
user_ip: :user_ip
# @return [Boolean] true if the Akismet API keys are configured
def self.enabled?
Danbooru.config.rakismet_key.present? && Danbooru.config.rakismet_url.present? && !Rails.env.test?
end
# rakismet raises an exception if the api key or url aren't configured
# @return [Boolean] true if the Akismet API keys are valid. Rakismet raises
# an exception if the API key or URL aren't configured
def self.working?
Rakismet.validate_key
rescue StandardError
false
end
# Check if the user seems like a spammer and should be banned. Checks if they
# have received more than 10 automatic spam reports in the last hour.
# @param user [User] the user to check
# @return [Boolean] true if the user should be autobanned
def self.is_spammer?(user)
return false if user.is_gold?
@@ -44,10 +54,15 @@ class SpamDetector
report_count >= AUTOBAN_THRESHOLD
end
# Autobans a user.
# @param spammer [User] the user to ban
def self.ban_spammer!(spammer)
spammer.bans.create!(banner: User.system, reason: "Spambot.", duration: AUTOBAN_DURATION)
end
# Initialize a spam check for a message.
# @param record [Dmail, ForumPost, Comment] the message to spam check
# @param user_ip [String] the IP address of the user who posted the message
def initialize(record, user_ip: nil)
case record
when Dmail
@@ -73,6 +88,9 @@ class SpamDetector
end
end
# Check if a message seems like spam. Gold users and users who have an account
# more than a month old aren't checked to reduce the number of API calls.
# @return [Boolean] true if the message is spam
def spam?
return false if !SpamDetector.enabled?
return false if user.is_gold?

12
app/logical/sqs_service.rb
View File

@@ -1,14 +1,25 @@
# A wrapper for the Amazon SQS API. Used by the PostArchive and PoolArchive
# service to record post and pool versions.
#
# @see https://docs.aws.amazon.com/sqs/index.html
# @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS/Client.html
class SqsService
attr_reader :url
# @param url [String] the URL of the Amazon SQS queue
def initialize(url)
@url = url
end
# @return [Boolean] true if the SQS service is configured
def enabled?
Danbooru.config.aws_credentials.set? && url.present?
end
# Sends a message to the Amazon SQS queue.
# @param string [String] the message to send
# @param options [Hash] extra options for the SQS call
# @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS/Client.html#send_message-instance_method
def send_message(string, options = {})
return unless enabled?
@@ -22,6 +33,7 @@ class SqsService
private
# @return [Aws::SQS::Client] the SQS API client object
def sqs
@sqs ||= Aws::SQS::Client.new(
credentials: Danbooru.config.aws_credentials,

39
app/logical/storage_manager.rb
View File

@@ -1,8 +1,22 @@
# StorageManager is an abstract superclass that defines a simple interface for
# storing files on local or remote backends. All image files stored by Danbooru
# are handled by a StorageManager.
#
# A StorageManager has methods for saving, deleting, and opening files, and for
# generates URLs for images.
#
# @abstract
# @see StorageManager::Local
# @see StorageManager::SFTP
class StorageManager
class Error < StandardError; end
attr_reader :base_url, :base_dir, :tagged_filenames
# Initialize a storage manager object.
# @param base_url [String] the base URL where images are stored (ex: "https://cdn.donmai.us/")
# @param base_dir [String] the base directory where images are stored (ex: "/var/www/danbooru/public/images")
# @param tagged_filenames [Boolean] whether image URLs can include tags
def initialize(base_url:, base_dir:, tagged_filenames: Danbooru.config.enable_seo_post_urls)
@base_url = base_url.chomp("/")
@base_dir = base_dir
@@ -13,33 +27,57 @@ class StorageManager
# location it should be overwritten atomically. Either the file is fully
# written, or an error is raised and the original file is left unchanged. The
# file should never be in a partially written state.
#
# @param io [IO] a file (or a readable IO object)
# @param path [String] the remote path where the file should be stored
def store(io, path)
raise NotImplementedError, "store not implemented"
end
# Delete the file at the given path. If the file doesn't exist, no error
# should be raised.
# @param path [String] the remote path of the file to be deleted
def delete(path)
raise NotImplementedError, "delete not implemented"
end
# Return a readonly copy of the file located at the given path.
# @param path [String] the remote path of the file to open
# @return [MediaFile] the image file
def open(path)
raise NotImplementedError, "open not implemented"
end
# Store or replace the given file belonging to the given post.
# @param io [IO] the file to store
# @param post [Post] the post the image belongs to
# @param type [Symbol] the image variant to store (:preview, :crop, :large, :original)
def store_file(io, post, type)
store(io, file_path(post.md5, post.file_ext, type))
end
# Delete the file belonging to the given post.
# @param post_id [Integer] the post's id
# @param md5 [String] the post's md5
# @param file_ext [String] the post's file extension
# @param type [Symbol] the image variant to delete (:preview, :crop, :large, :original)
def delete_file(post_id, md5, file_ext, type)
delete(file_path(md5, file_ext, type))
end
# Return a readonly copy of the image belonging to the given post.
# @param post [Post] the post
# @param type [Symbol] the image variant to open (:preview, :crop, :large, :original)
# @return [MediaFile] the image file
def open_file(post, type)
self.open(file_path(post.md5, post.file_ext, type))
end
# Generate the image URL for the given post.
# @param post [Post] the post
# @param type [Symbol] the post's image variant (:preview, :crop, :large, :original)
# @param tagged_filename [Boolean] whether the URL should contain the post's tags
# @return [String] the image URL
def file_url(post, type, tagged_filenames: false)
subdir = subdir_for(post.md5)
file = file_name(post.md5, post.file_ext, type)
@@ -100,6 +138,7 @@ class StorageManager
"#{md5[0..1]}/#{md5[2..3]}/"
end
# Generate the tags in the image URL.
def seo_tags(post)
return "" if !tagged_filenames

5
app/logical/storage_manager/rclone.rb
View File

@@ -1,3 +1,8 @@
# A StorageManager that stores file on remote filesystem using rclone. Rclone
# can store files on most cloud storage systems. Requires the `rclone` binary to
# be installed and configured.
#
# @see https://rclone.org/
class StorageManager::Rclone < StorageManager
class Error < StandardError; end
attr_reader :remote, :bucket, :rclone_path, :rclone_options

1
app/logical/storage_manager/sftp.rb
View File

@@ -1,3 +1,4 @@
# A StorageManager that stores files on a remote filesystem using SFTP.
class StorageManager::SFTP < StorageManager
DEFAULT_PERMISSIONS = 0o644

57
app/logical/table_builder.rb
View File

@@ -1,7 +1,38 @@
# A helper class for building HTML tables. Used in views.
#
# @example
# <%= table_for @tags do |table| %>
# <% table.column :name do |tag| %>
# <%= link_to_wiki "?", tag.name %>
# <%= link_to tag.name, posts_path(tags: tag.name) %>
# <% end %>
# <% table.column :post_count %>
# <% end %>
#
# @see app/views/table_builder/_table.html.erb
class TableBuilder
# Represents a single column in the table.
class Column
attr_reader :attribute, :name, :block, :header_attributes, :body_attributes
# Define a table column.
#
# @example
# <% table.column :post_count %>
#
# @example
# <% table.column :name do |tag| %>
# <%= tag.pretty_name %>
# <% end %>
#
# @param attribute [Symbol] The attribute in the model the column is for.
# The column's name and value will come from this attribute by default.
# @param name [String] the column's name, if different from the attribute name.
# @param column [String] the column name
# @param th [Hash] the HTML attributes for the column's <th> tag.
# @param td [Hash] the HTML attributes for the column's <td> tag.
# @param width [String] the HTML width value for the <th> tag.
# @yieldparam item a block that returns the column value based on the item.
def initialize(attribute = nil, column: nil, th: {}, td: {}, width: nil, name: nil, &block)
@attribute = attribute
@column = column
@@ -23,6 +54,11 @@ class TableBuilder
end
end
# Returns the value of the table cell.
# @param item [ApplicationRecord] the table cell item
# @param i [Integer] the table row number
# @param j [Integer] the table column number
# @return [#to_s] the value of the table cell
def value(item, i, j)
if block.present?
block.call(item, i, j, self)
@@ -37,6 +73,20 @@ class TableBuilder
attr_reader :columns, :table_attributes, :row_attributes, :items
# Build a table for an array of objects, one object per row.
#
# The <table> tag is automatically given an HTML id of the form `{name}-table`.
# For example, `posts-table`, `tags-table`.
#
# The <tr> tag is automatically given an HTML id of the form `{name}-{id}`.
# For example, `post-1234`, `tag-4567`, etc. Each <tr> tag also gets a set of
# data attributes for each model; see #html_data_attributes in app/policies.
#
# @param items [Array<ApplicationRecord>] The list of ActiveRecord objects to
# build the table for. One item per table row.
# @param tr [Hash] optional HTML attributes for the <tr> tag for each row
# @param table_attributes [Hash] optional HTML attributes for the <table> tag
# @yieldparam table [self] the table being built
def initialize(items, tr: {}, **table_attributes)
@items = items
@columns = []
@@ -50,10 +100,17 @@ class TableBuilder
yield self if block_given?
end
# Add a column to the table.
# @example
# table.column(:name)
def column(...)
@columns << Column.new(...)
end
# Return the HTML attributes for each <tr> tag.
# @param item [ApplicationRecord] the item for this row
# @param i [Integer] the row number (unused)
# @return [Hash] the <tr> attributes
def all_row_attributes(item, i)
return {} if !item.is_a?(ApplicationRecord)

2
app/logical/tag_category.rb
View File

@@ -1,3 +1,5 @@
# Utility methods for working with tag categories (general, character,
# copyright, artist, meta).
class TagCategory
module Mappings
# Returns a hash mapping various tag categories to a numerical value.

26
app/logical/tag_mover.rb
View File

@@ -1,12 +1,21 @@
# Rename a tag and move everything associated with it. Used when renaming or
# aliasing a tag in a bulk update request. Moves everything associated with the
# tag, including aliases, implications, *_(cosplay) and *_(style) tags, artist
# and wiki pages, saved searches, blacklists, and retags the posts themselves.
class TagMover
attr_reader :old_tag, :new_tag, :user
# Initalize a tag move.
# @param old_name [String] the name of the tag to move
# @param new_name [String] the new tag name
# @param user [User] the user to credit for all edits in moving the tag
def initialize(old_name, new_name, user: User.system)
@old_tag = Tag.find_or_create_by_name(old_name)
@new_tag = Tag.find_or_create_by_name(new_name)
@user = user
end
# Perform the tag move.
def move!
CurrentUser.scoped(user) do
move_tag_category!
@@ -23,6 +32,7 @@ class TagMover
end
end
# Sync the category of both tags, if one is a general tag and the other is non-general.
def move_tag_category!
if old_tag.general? && !new_tag.general?
old_tag.update!(category: new_tag.category)
@@ -31,6 +41,8 @@ class TagMover
end
end
# Move the artist entry from the old tag to the new tag, merging it into the
# new tag's artist entry if it already has an artist entry.
def move_artist!
return unless old_tag.artist? && old_artist.present? && !old_artist.is_deleted?
@@ -43,6 +55,8 @@ class TagMover
end
end
# Move the wiki from the old tag to the new tag, merging it into the new tag's
# wiki if it already has a wiki.
def move_wiki!
return unless old_wiki.present? && !old_wiki.is_deleted?
@@ -53,6 +67,7 @@ class TagMover
end
end
# Retag the posts from the old tag to the new tag.
def move_posts!
Post.raw_tag_match(old_tag.name).find_each do |post|
post.lock!
@@ -62,12 +77,14 @@ class TagMover
end
end
# Transfer any aliases pointing to the old tag to point to the new tag.
def move_aliases!
old_tag.consequent_aliases.each do |tag_alias|
tag_alias.update!(consequent_name: new_tag.name)
end
end
# Transfer any implications from the old tag to the new tag.
def move_implications!
old_tag.antecedent_implications.each do |tag_implication|
tag_implication.update!(antecedent_name: new_tag.name)
@@ -78,6 +95,7 @@ class TagMover
end
end
# Move the character's *_(cosplay) tag if it exists.
def move_cosplay_tag!
old_cosplay_tag = "#{old_tag.name}_(cosplay)"
new_cosplay_tag = "#{new_tag.name}_(cosplay)"
@@ -87,6 +105,7 @@ class TagMover
end
end
# Move the artist's *_(style) tag if it exists.
def move_style_tag!
old_style_tag = "#{old_tag.name}_(style)"
new_style_tag = "#{new_tag.name}_(style)"
@@ -96,18 +115,24 @@ class TagMover
end
end
# Update all saved searches to use the new tag.
def move_saved_searches!
SavedSearch.rewrite_queries!(old_tag.name, new_tag.name)
end
# Update all blacklists to use the new tag.
def move_blacklists!
User.rewrite_blacklists!(old_tag.name, new_tag.name)
end
# Update any wiki pages linking to the old tag, to link to the new tag.
def rewrite_wiki_links!
WikiPage.rewrite_wiki_links!(old_tag.name, new_tag.name)
end
# Merge two artist entries, copying everything from the old entry to the new
# one. Duplicate information will be automatically stripped when the artist
# is saved.
def merge_artists!
old_artist.lock!
new_artist.lock!
@@ -126,6 +151,7 @@ class TagMover
old_artist.save!
end
# Merge the other names from both wikis, then mark the old wiki as deleted.
def merge_wikis!
old_wiki.lock!
new_wiki.lock!

15
app/logical/twitter_api_client.rb
View File

@@ -1,22 +1,37 @@
# A simple Twitter API client that can authenticate to Twitter and fetch tweets by ID.
# @see https://developer.twitter.com/en/docs/getting-started
class TwitterApiClient
extend Memoist
attr_reader :api_key, :api_secret
# Create a Twitter API client
# @param api_key [String] the Twitter API key
# @param api_secret [String] the Twitter API secret
def initialize(api_key, api_secret)
@api_key, @api_secret = api_key, api_secret
end
# Authenticate to Twitter with an API key and secret and receive a bearer token in response.
# @param token_expiry [Integer] the number of seconds to cache the token
# @return [String] the Twitter bearer token
# @see https://developer.twitter.com/en/docs/authentication/api-reference/token
def bearer_token(token_expiry = 24.hours)
http = Danbooru::Http.basic_auth(user: api_key, pass: api_secret)
response = http.cache(token_expiry).post("https://api.twitter.com/oauth2/token", form: { grant_type: :client_credentials })
response.parse["access_token"]
end
# @return [Danbooru::Http] the HTTP client to connect to Twitter with
def client
Danbooru::Http.auth("Bearer #{bearer_token}")
end
# Fetch a tweet by id.
# @param id [Integer] the Twitter tweet id
# @param cache [Integer] the number of seconds to cache the response
# @return [Object] the tweet
# @see https://developer.twitter.com/en/docs/twitter-api/v1/tweets/post-and-engage/api-reference/get-statuses-show-id
def status(id, cache: 1.minute)
response = client.cache(cache).get("https://api.twitter.com/1.1/statuses/show.json?id=#{id}&tweet_mode=extended")
response.parse.with_indifferent_access

58
app/logical/upload_limit.rb
View File

@@ -1,3 +1,24 @@
# Calculates a user's upload limit:
#
# * Each status:pending post takes up one upload slot.
# * Each status:appealed post takes up three upload slots.
# * If any of your uploads are manually deleted in under three days (status:deleted age:<3d), they take up five upload slots.
# * Slots are freed when uploads are approved or deleted.
# * You start out with 15 upload slots and can have between 5 and 40 upload slots.
# * You gain and lose slots based on approvals and deletions:
# ** You lose a slot for every 3 deletions.
# ** You gain a slot for every N approved uploads, where N depends on how many slots you already have:
# *** If you have 15 slots or less, then you gain a slot for every 10 approved uploads.
# *** If you have more than 15 slots, then you need 10 approvals, plus 2 more per upload slot over 15.
#
# Internally, upload limits are based on a point system, where you start with
# 1000 points and level up to 10,000 points. Each approval is worth 10 points
# and each deletion costs 1/3 of the points to the next level. Points are mapped
# to levels such that each level costs 20 more points (2 approvals) than the
# last. Levels are mapped to upload slots such that levels range from 0 - 35,
# and upload slots range from 5 - 40.
#
# @see https://danbooru.donmai.us/wiki_pages/about:upload_limits
class UploadLimit
extend Memoist
@@ -8,18 +29,24 @@ class UploadLimit
attr_reader :user
# Create an upload limit object for a user.
# @param user [User]
def initialize(user)
@user = user
end
# @return [Boolean] true if the user can't upload because they're out of upload slots.
def limited?
!user.can_upload_free? && used_upload_slots >= upload_slots
end
# @return [Boolean] true if the user is at max level.
def maxed?
user.upload_points >= MAXIMUM_POINTS
end
# @return [Integer] The number of upload slots in use. Pending posts take 1
# slot, appeals take 3, and early deletions take 5.
def used_upload_slots
pending_count = user.posts.pending.count
appealed_count = user.post_appeals.pending.count
@@ -28,26 +55,38 @@ class UploadLimit
pending_count + (early_deleted_count * DELETION_COST) + (appealed_count * APPEAL_COST)
end
# @return [Integer] The number of unused upload slots, that is, the number of
# posts the user can upload.
def free_upload_slots
upload_slots - used_upload_slots
end
# @return [Integer] The user's total number of upload slots. Ranges from 5 to 40.
def upload_slots
upload_level + 5
end
# @return [Integer] The user's current upload level. Ranges from 0 to 35.
def upload_level
UploadLimit.points_to_level(user.upload_points)
end
# @return [Integer] The number of approvals received so far on the current level.
def approvals_on_current_level
(user.upload_points - UploadLimit.level_to_points(upload_level)) / 10
end
# @return [Integer] The number of approvals needed to reach the next level.
def approvals_for_next_level
UploadLimit.points_for_next_level(upload_level) / 10
end
# Update the uploader's upload points when a post is approved or deleted.
# @param post [Post] The post that was approved or deleted.
# @param incremental [Boolean] True if the post was status:pending and we can
# simply increment/decrement the upload points. False if the post was an
# approval or undeletion of an old post, and we have to replay the user's
# entire upload history to recalculate their points.
def update_limit!(post, incremental: true)
return if user.can_upload_free?
@@ -62,6 +101,9 @@ class UploadLimit
end
end
# Recalculate the user's upload points based on replaying their entire upload history.
# @param user [User] the user
# @return [Integer] the user's upload points
def self.points_for_user(user)
points = INITIAL_POINTS
@@ -76,6 +118,12 @@ class UploadLimit
points
end
# Calculate the value of a approval or deletion. Approvals are worth a fixed
# 10 points. Deletions cost 1/3 of the points needed for the next level.
#
# @param current_points [Integer] the user's current number of upload points
# @param is_deleted [Boolean] whether the post was deleted or approved
# @return [Integer] the number of points this upload is worth
def self.upload_value(current_points, is_deleted)
if is_deleted
level = points_to_level(current_points)
@@ -85,10 +133,17 @@ class UploadLimit
end
end
# Calculate the number of upload points needed to reach the next upload level.
# This is the number of approvals needed, times 10.
# @param level [Integer] the current upload level
# @return [Integer] The number of points needed to reach the next upload level
def self.points_for_next_level(level)
100 + 20 * [level - 10, 0].max
end
# Calculate the level that corresponds to a given number of upload points.
# @param [Integer] the upload points (0 - 10,000)
# @return [Integer] the upload level (0 - 35)
def self.points_to_level(points)
level = 0
@@ -101,6 +156,9 @@ class UploadLimit
level
end
# Calculate the base upload points that correspond to a given upload level.
# @param [Integer] the upload level (0 - 35)
# @return [Integer] the upload points (0 - 10,000)
def self.level_to_points(level)
(1..level).map do |n|
points_for_next_level(n - 1)

1
app/logical/upload_service.rb
View File

@@ -1,3 +1,4 @@
# A service object for uploading an image.
class UploadService
attr_reader :params, :post, :upload

11
app/logical/user_deletion.rb
View File

@@ -1,3 +1,7 @@
# Delete a user's account. Deleting an account really just deactivates the
# account, it doesn't fully delete the user from the database. It wipes their
# username, password, account settings, favorites, and saved searches, and logs
# the deletion.
class UserDeletion
include ActiveModel::Validations
@@ -5,12 +9,19 @@ class UserDeletion
validate :validate_deletion
# Initialize a user deletion.
# @param user [User] the user to delete
# @param password [String] the user's password (for confirmation)
# @param request the HTTP request (for logging the deletion in the user event log)
def initialize(user, password, request)
@user = user
@password = password
@request = request
end
# Delete the account, if the deletion is allowed.
# @return [Boolean] if the deletion failed
# @return [User] if the deletion succeeded
def delete!
return false if invalid?

11
app/logical/user_promotion.rb
View File

@@ -1,6 +1,16 @@
# Promotes the user to a higher level, or grants them approver or unlimited
# uploader privileges. Also validates that the promotion is allowed, gives the
# user a feedback, sends them a notification dmail, and creates a mod action for
# the promotion.
class UserPromotion
attr_reader :user, :promoter, :new_level, :old_can_approve_posts, :old_can_upload_free, :can_upload_free, :can_approve_posts
# Initialize a new promotion.
# @param user [User] the user to promote
# @param promoter [User] the user doing the promotion
# @param new_level [Integer] the new user level
# @param can_upload_free [Boolean] whether the user should gain unlimited upload privileges
# @param can_approve_posts [Boolean] whether the user should gain approval privileges
def initialize(user, promoter, new_level, can_upload_free: nil, can_approve_posts: nil)
@user = user
@promoter = promoter
@@ -54,6 +64,7 @@ class UserPromotion
end
end
# Build the dmail and user feedback message.
def build_messages
messages = []

13
app/logical/user_verifier.rb
View File

@@ -1,15 +1,21 @@
# Checks whether a new account seems suspicious and should require email verification.
# Checks whether a new account requires verification. An account requires
# verification if the IP is a proxy, or the IP is under a partial (signup) IP
# ban, or it was used to create another account recently.
class UserVerifier
extend Memoist
attr_reader :current_user, :request
# current_user is the user creating the new account, not the new account itself.
# Create a user verifier.
# @param current_user [User] the user creating the new account, not the new account itself.
# @param request the HTTP request
def initialize(current_user, request)
@current_user, @request = current_user, request
end
# Returns true if the new account should be restricted. Signups from local
# IPs are unrestricted so that verification isn't required for development,
# testing, or personal boorus.
def requires_verification?
return false if !Danbooru.config.new_user_verification?
return false if ip_address.is_local?
@@ -18,6 +24,7 @@ class UserVerifier
is_ip_banned? || is_logged_in? || is_recent_signup? || is_proxy?
end
# @return [Integer] Returns whether the new account should be Restricted or a Member
def initial_level
if requires_verification?
User::Levels::RESTRICTED

9
app/logical/validating_socket.rb
View File

@@ -1,6 +1,9 @@
# A TCPSocket wrapper that disallows connections to local or private IPs. Used for SSRF protection.
# https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
# A TCPSocket wrapper that disallows connections to local or private IPs. Used
# by {Danbooru::Http} to prevent server-side request forgery (SSRF) attacks. For
# example, if we try to download an image from http://example.com/image.jpg, but
# example.com resolves to 127.0.0.1, then the request is prohibited.
#
# @see https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
require "resolv"
class ValidatingSocket < TCPSocket

4
app/mailers/user_mailer.rb
View File

@@ -2,21 +2,25 @@ class UserMailer < ApplicationMailer
helper :application
helper :users
# The email sent when a user receives a DMail.
def dmail_notice(dmail)
@dmail = dmail
mail to: dmail.to.email_with_name, subject: "#{Danbooru.config.app_name} - Message received from #{dmail.from.name}"
end
# The email sent when a user requests a password reset.
def password_reset(user)
@user = user
mail to: @user.email_with_name, subject: "#{Danbooru.config.app_name} password reset request"
end
# The email sent when a user changes their email address.
def email_change_confirmation(user)
@user = user
mail to: @user.email_with_name, subject: "Confirm your email address"
end
# The email sent when a new user signs up with an email address.
def welcome_user(user)
@user = user
mail to: @user.email_with_name, subject: "Welcome to #{Danbooru.config.app_name}! Confirm your email address"