docs: add documentation for various classes in app/logical.

This commit is contained in:
evazion
2021-06-23 05:09:55 -05:00
parent e5cfb7904c
commit ed302fdf4d
33 changed files with 518 additions and 25 deletions

View File

@@ -1,3 +1,4 @@
# Parse an PNG and return whether or not it's animated.
class APNGInspector
attr_reader :frames

View File

@@ -1,5 +1,8 @@
# https://github.com/plataformatec/responders
# https://github.com/plataformatec/responders/blob/master/lib/action_controller/responder.rb
# Hooks into `respond_with` to add some custom behavior, including support for
# the `expires_in`, `expiry`, and `only` params.
#
# @see https://github.com/plataformatec/responders
# @see https://github.com/plataformatec/responders/blob/master/lib/action_controller/responder.rb
class ApplicationResponder < ActionController::Responder
# this is called by respond_with for non-html, non-js responses.
def to_format

View File

@@ -1,9 +1,17 @@
# Demote all approvers who haven't approved at least 30 posts in the last 45
# days. Moderators and recently promoted approvers are exempt. Runs as a monthly
# maintenance task. Approvers who are facing demotion are sent a weekly warning
# dmail first.
#
# @see DanbooruMaintenance#monthly
module ApproverPruner
module_function
APPROVAL_PERIOD = 45.days
MINIMUM_APPROVALS = 30
# Get the list of inactive approvers.
# @return [Array<User>] the list of inactive approvers
def inactive_approvers
approvers = User.bit_prefs_match(:can_approve_posts, true)
approvers = approvers.where("level < ?", User::Levels::MODERATOR)
@@ -16,9 +24,10 @@ module ApproverPruner
end
end
# Demote all inactive approvers
def prune!
inactive_approvers.each do |user|
CurrentUser.scoped(User.system, "127.0.0.1") do
CurrentUser.scoped(User.system) do
user.update!(can_approve_posts: false)
user.feedback.create(category: "neutral", body: "Lost approval privileges", creator: User.system)
@@ -31,6 +40,7 @@ module ApproverPruner
end
end
# Send a warning dmail to approvers who are pending demotion.
def dmail_inactive_approvers!
days_until_next_month = (Date.current.next_month.beginning_of_month - Date.current).to_i
return unless days_until_next_month <= 21

View File

@@ -1,3 +1,4 @@
# Find the artist entry for a given artist profile URL.
module ArtistFinder
module_function
@@ -131,6 +132,14 @@ module ArtistFinder
%r{\Ahttps?://(?:[a-zA-Z0-9_-]+\.)*#{domain}/\z}i
end)
# Find the artist for a given artist profile URL. May return multiple Artists
# in the event of duplicate artist entries.
#
# Uses a path-stripping algorithm to find any artist URL that is a prefix
# of the given URL. A site blacklist is used to prevent false positives.
#
# @param url [String] the artist profile URL
# @return [Array<Artist>] the list of matching artists
def find_artists(url)
url = ArtistUrl.normalize(url)
artists = []

View File

@@ -1,3 +1,10 @@
# Autocomplete tags, usernames, pools, and more.
#
# @example
# AutocompleteService.new("touho", :tag).autocomplete_results
# #=> [{ type: :tag, label: "touhou", value: "touhou", category: 3, post_count: 42 }]
#
# @see AutocompleteController
class AutocompleteService
extend Memoist
@@ -18,6 +25,11 @@ class AutocompleteService
attr_reader :query, :type, :limit, :current_user
# Perform completion for the given search type and query.
# @param query [String] the string being completed
# @param type [String] the type of completion being performed
# @param current_user [User] the user we're performing completion for
# @param limit [Integer] the max number of results to return
def initialize(query, type, current_user: User.anonymous, limit: 10)
@query = query.to_s
@type = type.to_s.to_sym
@@ -25,6 +37,8 @@ class AutocompleteService
@limit = limit
end
# Return the results of the completion.
# @return [Array<Hash>] the autocomplete results
def autocomplete_results
case type
when :tag_query
@@ -52,6 +66,9 @@ class AutocompleteService
end
end
# Complete a tag search (a regular tag or a metatag)
# @param string [String] the string to complete
# @return [Array<Hash>] the autocomplete results
def autocomplete_tag_query(string)
term = PostQueryBuilder.new(string).terms.first
return [] if term.nil?
@@ -64,10 +81,20 @@ class AutocompleteService
end
end
# Find tags matching a search.
#
# If the string is non-English, translate it to a Danbooru tag.
# If the string is a slash abbreviation, expand the abbreviation.
# If the string has a wildcard, do a wildcard search.
# If the string doesn't match anything, perform autocorrect.
#
# @param string [String] the string to complete
# @return [Array<Hash>] the autocomplete results
def autocomplete_tag(string)
return [] if string.size > TagNameValidator::MAX_TAG_LENGTH
return [] if string.start_with?("http://", "https://")
# XXX convert to NFKC? deaccent?
if !string.ascii_only?
results = tag_other_name_matches(string)
elsif string.starts_with?("/")
@@ -90,6 +117,9 @@ class AutocompleteService
results
end
# Find tags or tag aliases matching a wildcard search.
# @param string [String] the string to complete
# @return [Array<Hash>] the autocomplete results
def tag_matches(string)
name_matches = Tag.nonempty.name_matches(string).order(post_count: :desc).limit(limit)
alias_matches = Tag.nonempty.alias_matches(string).order(post_count: :desc).limit(limit)
@@ -103,6 +133,12 @@ class AutocompleteService
end
end
# Find tags matching a slash abbreviation.
# Example: /evth => eyebrows_visible_through_hair
#
# @param string [String] the string to complete
# @param max_length [Integer] the max abbreviation length
# @return [Array<Hash>] the autocomplete results
def tag_abbreviation_matches(string, max_length: 10)
return [] if string.size > max_length
@@ -113,6 +149,11 @@ class AutocompleteService
end
end
# Find tags matching a mispelled tag.
# Example: logn_hair => long_hair
#
# @param string [String] the string to complete
# @return [Array<Hash>] the autocomplete results
def tag_autocorrect_matches(string)
# autocorrect uses trigram indexing, which needs at least 3 alphanumeric characters to work.
return [] if string.remove(/[^a-zA-Z0-9]/).size < 3
@@ -124,6 +165,12 @@ class AutocompleteService
end
end
# Find tags matching a non-English string. Does a `name*` search in wiki page
# and artist other names to translate the non-English tag to a Danbooru tag.
# Example: 東方 => touhou.
#
# @param string [String] the string to complete
# @return [Array<Hash>] the autocomplete results
def tag_other_name_matches(string)
artists = Artist.undeleted.where_any_in_array_starts_with(:other_names, string)
wikis = WikiPage.undeleted.where_any_in_array_starts_with(:other_names, string)
@@ -137,6 +184,10 @@ class AutocompleteService
end
end
# Complete a metatag.
# @param metatag [String] the type of metatag to complete
# @param value [String] the value of the metatag
# @return [Array<Hash>] the autocomplete results
def autocomplete_metatag(metatag, value)
results = case metatag.to_sym
when :user, :approver, :commenter, :comm, :noter, :noteupdater, :commentaryupdater,
@@ -159,6 +210,10 @@ class AutocompleteService
end
end
# Complete a static metatag: rating, filetype, etc.
# @param metatag [String] the type of metatag to complete
# @param value [String] the value of the metatag
# @return [Array<Hash>] the autocomplete results
def autocomplete_static_metatag(metatag, value)
values = STATIC_METATAGS[metatag.to_sym]
results = values.select { |v| v.starts_with?(value.downcase) }.sort.take(limit)
@@ -168,6 +223,9 @@ class AutocompleteService
end
end
# Complete a pool name. Does a `*name*` search.
# @param string [String] the name of the pool
# @return [Array<Hash>] the autocomplete results
def autocomplete_pool(string)
string = "*" + string + "*" unless string.include?("*")
pools = Pool.undeleted.name_matches(string).search(order: "post_count").limit(limit)
@@ -177,6 +235,9 @@ class AutocompleteService
end
end
# Complete a favorite group name. Does a `*name*` search.
# @param string [String] the name of the favgroup
# @return [Array<Hash>] the autocomplete results
def autocomplete_favorite_group(string)
string = "*" + string + "*" unless string.include?("*")
favgroups = FavoriteGroup.visible(current_user).where(creator: current_user).name_matches(string).search(order: "post_count").limit(limit)
@@ -186,6 +247,9 @@ class AutocompleteService
end
end
# Complete a saved search label. Does a `*name*` search.
# @param string [String] the name of the label
# @return [Array<Hash>] the autocomplete results
def autocomplete_saved_search_label(string)
string = "*" + string + "*" unless string.include?("*")
labels = current_user.saved_searches.labels_like(string).take(limit)
@@ -195,6 +259,9 @@ class AutocompleteService
end
end
# Complete an artist name. Does a `name*` search.
# @param string [String] the name of the artist
# @return [Array<Hash>] the autocomplete results
def autocomplete_artist(string)
string = string + "*" unless string.include?("*")
artists = Artist.undeleted.name_matches(string).search(order: "post_count").limit(limit)
@@ -204,6 +271,9 @@ class AutocompleteService
end
end
# Complete a wiki name. Does a `name*` search.
# @param string [String] the name of the wiki
# @return [Array<Hash>] the autocomplete results
def autocomplete_wiki_page(string)
string = string + "*" unless string.include?("*")
wiki_pages = WikiPage.undeleted.title_matches(string).search(order: "post_count").limit(limit)
@@ -213,6 +283,9 @@ class AutocompleteService
end
end
# Complete a user name. Does a `name*` search.
# @param string [String] the name of the user
# @return [Array<Hash>] the autocomplete results
def autocomplete_user(string)
string = string + "*" unless string.include?("*")
users = User.search(name_matches: string, current_user_first: true, order: "post_upload_count").limit(limit)
@@ -222,17 +295,27 @@ class AutocompleteService
end
end
# Complete an @mention for a user name. Does a `name*` search.
# @param string [String] the name of the user
# @return [Array<Hash>] the autocomplete results
def autocomplete_mention(string)
autocomplete_user(string).map do |result|
{ **result, value: "@" + result[:value] }
end
end
# Complete a search typed in the browser address bar.
# @param string [String] the name of the tag
# @return [Array<(String, [Array<String>])>] the autocomplete results
# @see https://en.wikipedia.org/wiki/OpenSearch
# @see https://developer.mozilla.org/en-US/docs/Web/OpenSearch
def autocomplete_opensearch(string)
results = autocomplete_tag(string).map { |result| result[:value] }
[query, results]
end
# How long autocomplete results can be cached. Cache short result lists (<10
# results) for less time because they're more likely to change.
def cache_duration
if autocomplete_results.size == limit
24.hours
@@ -241,6 +324,7 @@ class AutocompleteService
end
end
# Whether the results can be safely cached with `Cache-Control: public`.
# Queries that don't depend on the current user are safe to cache publicly.
def cache_publicly?
if type == :tag_query && parsed_search&.type == :tag

View File

@@ -1,22 +1,40 @@
# Export all public data in a model to BigQuery and to Google Cloud Storage.
# Perform a daily database dump to BigQuery and to Google Cloud Storage. This
# contains all data visible to anonymous users.
#
# The database dumps are publicly accessible. The BigQuery data is at
# `danbooru1.danbooru_public.{table}`. The Google Cloud Storage data is at
# `gs://danbooru_public/data/{table}.json`. The storage bucket contains the data
# in newline-delimited JSON format.
#
# @see DanbooruMaintenance#daily
# @see https://console.cloud.google.com/storage/browser/danbooru_public
# @see https://console.cloud.google.com/bigquery?d=danbooru_public&p=danbooru1&t=posts&page=table
# @see https://cloud.google.com/bigquery/docs
# @see https://cloud.google.com/storage/docs
# @see https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON
class BigqueryExportService
extend Memoist
attr_reader :model, :dataset_name, :credentials
# Prepare to dump a table. Call {#export!} to dump it.
# @param model [ApplicationRecord] the database table to dump
# @param dataset_name [String] the BigQuery dataset name
# @param credentials [String] the Google Cloud credentials (in JSON format)
def initialize(model = nil, dataset_name: "danbooru_public", credentials: default_credentials)
@model = model
@dataset_name = dataset_name
@credentials = credentials
end
# Start a background job for each table to export it to BigQuery.
def self.async_export_all!(**options)
models.each do |model|
BigqueryExportJob.perform_later(model: model, **options)
end
end
# The list of database tables to dump.
def self.models
Rails.application.eager_load!
@@ -29,6 +47,7 @@ class BigqueryExportService
credentials.present?
end
# Dump the table to Cloud Storage and BigQuery.
def export!
return unless enabled? && records.any?
@@ -36,7 +55,7 @@ class BigqueryExportService
upload_to_bigquery!(file)
end
# Dump the model records to a gzipped, newline-delimited JSON tempfile.
# Dump the table's records to a gzipped, newline-delimited JSON tempfile.
def dump_records!
file = Tempfile.new("danbooru-export-dump-", binmode: true)
file = Zlib::GzipWriter.new(file)
@@ -51,8 +70,7 @@ class BigqueryExportService
file
end
# GCS: gs://danbooru_public/data/{model}.json
# BQ: danbooru1.danbooru_public.{model}
# Upload the JSON dump to Cloud Storage, then load it into BigQuery.
def upload_to_bigquery!(file)
table_name = model.model_name.collection
gsfilename = "data/#{table_name}.json"
@@ -64,24 +82,27 @@ class BigqueryExportService
job
end
# private
# The list of records to dump.
def records
model.visible(User.anonymous)
end
# Find or create the BigQuery dataset.
def dataset
bigquery.dataset(dataset_name) || bigquery.create_dataset(dataset_name)
end
# Find or create the Google Storage bucket.
def bucket
storage.bucket(dataset_name) || storage.create_bucket(dataset_name, acl: "public", default_acl: "public", storage_class: "standard", location: "us-east1")
end
# The BigQuery API client.
def bigquery
Google::Cloud::Bigquery.new(credentials: credentials)
end
# The Cloud Storage API client.
def storage
Google::Cloud::Storage.new(credentials: credentials)
end

View File

@@ -1,3 +1,5 @@
# Process a bulk update request. Parses the request and applies each line in
# sequence.
class BulkUpdateRequestProcessor
# Maximum tag size allowed by the rename command before an alias must be used.
MAXIMUM_RENAME_COUNT = 200
@@ -17,6 +19,7 @@ class BulkUpdateRequestProcessor
validate :validate_script
validate :validate_script_length
# @param bulk_update_request [String] the BUR
def initialize(bulk_update_request)
@bulk_update_request = bulk_update_request
end
@@ -50,6 +53,8 @@ class BulkUpdateRequestProcessor
end
end
# Validate the bulk update request when it is created or approved.
#
# validation_context will be either :request (when the BUR is first created
# or edited) or :approval (when the BUR is approved). Certain validations
# only run when the BUR is requested, not when it's approved.
@@ -117,12 +122,15 @@ class BulkUpdateRequestProcessor
end
end
# Validate that the script isn't too long.
def validate_script_length
if commands.size > MAXIMUM_SCRIPT_LENGTH
errors.add(:base, "Bulk update request is too long (maximum size: #{MAXIMUM_SCRIPT_LENGTH} lines). Split your request into smaller chunks and try again.")
end
end
# Apply the script.
# @param approver [User] the approver of the request
def process!(approver)
ActiveRecord::Base.transaction do
commands.map do |command, *args|
@@ -162,6 +170,8 @@ class BulkUpdateRequestProcessor
end
end
# The list of tags in the script. Used for search BURs by tag.
# @return [Tag] the list of tags
def affected_tags
commands.flat_map do |command, *args|
case command
@@ -178,6 +188,7 @@ class BulkUpdateRequestProcessor
end.sort.uniq
end
# Returns true if a non-Admin is allowed to approve a rename or alias request.
def is_tag_move_allowed?
commands.all? do |command, *args|
case command
@@ -194,6 +205,8 @@ class BulkUpdateRequestProcessor
end
end
# Convert the BUR to DText format.
# @return [String]
def to_dtext
commands.map do |command, *args|
case command

View File

@@ -1,6 +1,8 @@
# Rejects bulk update requests that haven't been approved in 60 days.
module BulkUpdateRequestPruner
module_function
# Posts a warning when a bulk update request is pending automatic rejection in 5 days.
def warn_old
BulkUpdateRequest.old.pending.find_each do |bulk_update_request|
if bulk_update_request.forum_topic
@@ -12,6 +14,7 @@ module BulkUpdateRequestPruner
end
end
# Rejects bulk update requests that haven't been approved in 60 days.
def reject_expired
BulkUpdateRequest.expired.pending.find_each do |bulk_update_request|
ApplicationRecord.transaction do

View File

@@ -1,4 +1,17 @@
# A wrapper around `Rails.cache` that adds some extra helper methods. All
# Danbooru code should use this class instead of using `Rails.cache` directly.
#
# In production, the cache is backed by Redis. In development, it's a temporary
# in-memory cache.
#
# @see config/initializers/cache_store.rb
# @see https://guides.rubyonrails.org/caching_with_rails.html#cache-stores
# @see https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html
class Cache
# Get multiple values from the cache at once.
# @param keys [Array<String>] the list of keys to fetch
# @param prefix [String] a prefix for each cache key
# @return [Hash<String, Object>] a map from cache keys to cache values
def self.get_multi(keys, prefix)
sanitized_key_to_key_hash = keys.map do |key|
["#{prefix}:#{Cache.hash(key)}", key]
@@ -14,28 +27,41 @@ class Cache
keys_to_values_hash
end
# Get a value from the cache. If the value isn't in the cache, use the block
# to generate the value.
# @param key [String] the key to fetch
# @param expiry_in_seconds [Integer] the lifetime of the cached value
# @param options [Hash] options to pass to Rails.cache.fetch
# @return the cached value
def self.get(key, expiry_in_seconds = nil, **options, &block)
Rails.cache.fetch(key, expires_in: expiry_in_seconds, **options, &block)
end
# Write a value to the cache.
# @param key [String] the key to cache
# @param value [Object] the value to cache
# @param expiry_in_seconds [Integer] the lifetime of the cached value
# @return the cached value
def self.put(key, value, expiry_in_seconds = nil)
Rails.cache.write(key, value, expires_in: expiry_in_seconds)
value
end
# Remove a value from the cache.
# @param key [String] the key to remove
def self.delete(key)
Rails.cache.delete(key)
nil
end
# Clear the entire cache.
def self.clear
Rails.cache.clear
end
def self.sanitize(key)
key.gsub(/\W/) {|x| "%#{x.ord}"}.slice(0, 230)
end
# Hash a cache key.
# @param string [String] the cache key to hash
# @return [String] the hashed key
def self.hash(string)
Digest::SHA256.base64digest(string)
end

View File

@@ -1,3 +1,5 @@
# A simple Cloudflare API client for purging cached images after they're
# regenerated or deleted.
class CloudflareService
attr_reader :api_token, :zone
@@ -9,6 +11,9 @@ class CloudflareService
api_token.present? && zone.present?
end
# Purge a list of URLs from Cloudflare's cache.
# @param urls [Array<String>] the list of URLs to purge
# @see https://api.cloudflare.com/#zone-purge-files-by-url
def purge_cache(urls)
return unless enabled?

View File

@@ -1,3 +1,18 @@
# Generates a set of CSS variables for site's color palette.
#
# The CSS variables are named like `var(--{hue}-{i})`, where `hue` is the color
# name (see below) and `i` is the brightless level (0-9, 0 is white and 9 is
# black).
#
# Based on the HSLuv color space. The color palette is designed to be
# perceptually uniform, that is, to have equal brightness levels across each
# color.
#
# @see https://www.hsluv.org/
# @see https://en.wikipedia.org/wiki/HSLuv
# @see https://github.com/hsluv/hsluv
# @see https://danbooru.donmai.us/static/colors
# @see app/javascript/src/styles/base/040_colors.css
module ColorPalette
module_function

View File

@@ -1,3 +1,15 @@
# A global variable containing the current user, the current IP address, the
# current user's country code, and whether safe mode is enabled.
#
# The current user is set during a request by {ApplicationController#set_current_user},
# which calls {SessionLoader#load}. The current user will not be set outside of
# the request cycle, for example, during background jobs, cron jobs, or on the
# console. For this reason, code outside of controllers and views, such as
# models, shouldn't rely on or assume the current user exists.
#
# @see https://api.rubyonrails.org/classes/ActiveSupport/CurrentAttributes.html
# @see ApplicationController#set_current_user
# @see SessionLoader#load
class CurrentUser < ActiveSupport::CurrentAttributes
attribute :user, :ip_addr, :country, :safe_mode
@@ -5,6 +17,10 @@ class CurrentUser < ActiveSupport::CurrentAttributes
delegate :id, to: :user, allow_nil: true
delegate_missing_to :user
# Run a block of code as another user.
# @param user [User] the user to run as
# @param ip_addr [String] the IP address to run as
# @yield the block
def self.scoped(user, ip_addr = "127.0.0.1", &block)
set(user: user, ip_addr: ip_addr) do
yield user

View File

@@ -1,20 +1,38 @@
require 'cgi'
require 'uri'
# The DText class handles Danbooru's markup language, DText. Parsing DText is
# handled by the DTextRagel class in the dtext_rb gem.
#
# @see https://github.com/evazion/dtext_rb
# @see https://danbooru.donmai.us/wiki_pages/help:dtext
class DText
MENTION_REGEXP = /(?<=^| )@\S+/
def self.format_text(text, data: nil, **options)
# Convert a string of DText to HTML.
# @param text [String] The DText input
# @param inline [Boolean] if true, allow only inline constructs. Ignore
# block-level constructs, such as paragraphs, quotes, lists, and tables.
# @param disable_mentions [Boolean] if true, don't generate @mentions.
# @param base_url [String, nil] if present, convert relative URLs to absolute URLs.
# @param data cached wiki/tag/artist data generated by {#preprocess}.
# @return [String, nil] The HTML output
def self.format_text(text, data: nil, inline: false, disable_mentions: false, base_url: nil)
return nil if text.nil?
data = preprocess([text]) if data.nil?
text = parse_embedded_tag_request(text)
html = DTextRagel.parse(text, **options)
html = DTextRagel.parse(text, inline: inline, disable_mentions: disable_mentions, base_url: base_url)
html = postprocess(html, *data)
html
rescue DTextRagel::Error
""
end
# Preprocess a set of DText messages and collect all tag, artist, and wiki
# page references. Called before rendering a collection of DText messages
# (e.g. comments or forum posts) to do all database lookups in one batch.
# @param [Array<String>] a list of DText strings
# @return an array of wiki pages, tags, and artists
def self.preprocess(dtext_messages)
dtext_messages = dtext_messages.map { |message| parse_embedded_tag_request(message) }
names = dtext_messages.map { |message| parse_wiki_titles(message) }.flatten.uniq
@@ -25,6 +43,11 @@ class DText
[wiki_pages, tags, artists]
end
# Rewrite the HTML produced by {#format_text} to colorize wiki links.
# @param wiki_pages [Array<WikiPage>]
# @param tags [Array<Tag>]
# @param artists [Array<Artist>]
# @return [String] the HTML output
def self.postprocess(html, wiki_pages, tags, artists)
fragment = Nokogiri::HTML.fragment(html)
@@ -69,11 +92,18 @@ class DText
fragment.to_s
end
# Wrap a DText message in a [quote] block.
# @param message [String] the DText to quote
# @param creator_name [String] the name of the user to quote.
# @return [String] the quoted DText
def self.quote(message, creator_name)
stripped_body = DText.strip_blocks(message, "quote")
"[quote]\n#{creator_name} said:\n\n#{stripped_body}\n[/quote]\n\n"
end
# Convert `[bur:<id>]`, `[ta:<id>]`, `[ti:<id>]` tags to DText.
# @param text [String] the DText input
# @return [String] the DText output
def self.parse_embedded_tag_request(text)
text = parse_embedded_tag_request_type(text, TagAlias, /\[ta:(?<id>\d+)\]/m)
text = parse_embedded_tag_request_type(text, TagImplication, /\[ti:(?<id>\d+)\]/m)
@@ -81,6 +111,11 @@ class DText
text
end
# Convert a `[bur:<id>]`, `[ta:<id>]`, or `[ti:<id>]` tag to DText.
# @param text [String] the DText input
# @param tag_request [BulkUpdateRequest, TagAlias, TagImplication]
# @param pattern [Regexp]
# @return [String] the DText output
def self.parse_embedded_tag_request_type(text, tag_request, pattern)
text.gsub(pattern) do |match|
obj = tag_request.find_by_id($~[:id])
@@ -88,6 +123,9 @@ class DText
end
end
# Convert a `[bur:<id>]`, `[ta:<id>]`, or `[ti:<id>]` tag to DText.
# @param obj [BulkUpdateRequest, TagAlias, TagImplication] the object to convert
# @return [String] the DText output
def self.tag_request_message(obj)
if obj.is_a?(TagRelationship)
if obj.is_active?
@@ -118,6 +156,9 @@ class DText
end
end
# Return a list of user names mentioned in a string of DText. Ignore mentions in [quote] blocks.
# @param text [String] the string of DText
# @return [Array<String>] the list of user names
def self.parse_mentions(text)
text = strip_blocks(text.to_s, "quote")
@@ -128,6 +169,9 @@ class DText
names.uniq
end
# Return a list of wiki pages mentioned in a string of DText.
# @param text [String] the string of DText
# @return [Array<String>] the list of wiki page names
def self.parse_wiki_titles(text)
html = DTextRagel.parse(text)
fragment = Nokogiri::HTML.fragment(html)
@@ -142,6 +186,9 @@ class DText
titles.uniq
end
# Return a list of external links mentioned in a string of DText.
# @param text [String] the string of DText
# @return [Array<String>] the list of external URLs
def self.parse_external_links(text)
html = DTextRagel.parse(text)
fragment = Nokogiri::HTML.fragment(html)
@@ -150,6 +197,10 @@ class DText
links.uniq
end
# Return whether the two strings of DText have the same set of links.
# @param a [String] a string of DText
# @param b [String] a string of DText
# @return [Boolean]
def self.dtext_links_differ?(a, b)
Set.new(parse_wiki_titles(a)) != Set.new(parse_wiki_titles(b)) ||
Set.new(parse_external_links(a)) != Set.new(parse_external_links(b))
@@ -159,6 +210,10 @@ class DText
# the capitalization of the old tag when rewriting it to the new tag, but if
# we can't determine how the new tag should be capitalized based on some
# simple heuristics, then we skip rewriting the tag.
# @param dtext [String] the DText input
# @param old_name [String] the old wiki name
# @param new_name [String] the new wiki name
# @return [String] the DText output
def self.rewrite_wiki_links(dtext, old_name, new_name)
old_name = old_name.downcase.squeeze("_").tr("_", " ").strip
new_name = new_name.downcase.squeeze("_").tr("_", " ").strip
@@ -201,6 +256,10 @@ class DText
end
end
# Remove all [<tag>] blocks from the DText.
# @param string [String] the DText input
# @param tag [String] the type of block to remove
# @return [String] the DText output
def self.strip_blocks(string, tag)
n = 0
stripped = ""
@@ -229,12 +288,18 @@ class DText
stripped.strip
end
# Remove all DText formatting from a string of DText, converting it to plain text.
# @param dtext [String] the DText input
# @return [String] the plain text output
def self.strip_dtext(dtext)
html = DTextRagel.parse(dtext)
text = to_plaintext(html)
text
end
# Remove all formatting from a string of HTML, converting it to plain text.
# @param html [String] the HTML input
# @return [String] the plain text output
def self.to_plaintext(html)
text = from_html(html) do |node|
case node.name
@@ -250,10 +315,16 @@ class DText
text.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
end
# Convert DText formatting to Markdown.
# @param dtext [String] the DText input
# @return [String] the Markdown output
def self.to_markdown(dtext)
html_to_markdown(format_text(dtext))
end
# Convert HTML to Markdown.
# @param html [String] the HTML input
# @return [String] the Markdown output
def self.html_to_markdown(html)
html = Nokogiri::HTML.fragment(html)
@@ -273,6 +344,10 @@ class DText
end.join
end
# Convert HTML to DText.
# @param html [String] the HTML input
# @param inline [Boolean] if true, convert <img> tags to plaintext
# @return [String] the DText output
def self.from_html(text, inline: false, &block)
html = Nokogiri::HTML.fragment(text)
@@ -336,13 +411,19 @@ class DText
dtext
end
# extract the first paragraph `needle` occurs in.
# Return the first paragraph the search string `needle` occurs in.
# @param needle [String] the string to search for
# @param dtext [String] the DText input
# @return [String] the first paragraph mentioning the search string
def self.extract_mention(dtext, needle)
dtext = dtext.gsub(/\r\n|\r|\n/, "\n")
excerpt = ActionController::Base.helpers.excerpt(dtext, needle, separator: "\n\n", radius: 1, omission: "")
excerpt
end
# Generate a short plain text excerpt from a DText string.
# @param length [Integer] the max length of the output
# @return [String] a plain text string
def self.excerpt(text, length: 160)
strip_dtext(text).split(/\r\n|\r|\n/).first.to_s.truncate(length)
end

View File

@@ -1,6 +1,13 @@
# The DanbooruLogger class handles logging messages to the Rails log and to NewRelic.
#
# @see https://guides.rubyonrails.org/debugging_rails_applications.html#the-logger
# @see https://docs.newrelic.com
class DanbooruLogger
HEADERS = %w[referer sec-fetch-dest sec-fetch-mode sec-fetch-site sec-fetch-user]
# Log a message to the Rails log and to NewRelic.
# @param message [String] the message to log
# @param params [Hash] optional key-value data to log with the message
def self.info(message, params = {})
Rails.logger.info(message)
@@ -10,6 +17,13 @@ class DanbooruLogger
end
end
# Log an exception to the Rails log and to NewRelic. The `expected` flag is
# used to separate expected exceptions, like search timeouts or auth failures,
# from unexpected exceptions, like runtime errors, in the NewRelic error log.
#
# @param message [Exception] the exception to log
# @param expected [Boolean] whether the exception was expected
# @param params [Hash] optional key-value data to log with the exception
def self.log(exception, expected: false, **params)
if expected
Rails.logger.info("#{exception.class}: #{exception.message}")
@@ -23,6 +37,12 @@ class DanbooruLogger
end
end
# Log extra HTTP request data to NewRelic. Logs the user's IP, user agent,
# request params, and session cookies.
#
# @param request the HTTP request
# @param session the Rails session
# @param user [User] the current user
def self.add_session_attributes(request, session, user)
add_attributes("request", { path: request.path })
add_attributes("request.headers", header_params(request))
@@ -31,6 +51,7 @@ class DanbooruLogger
add_attributes("user", user_params(request, user))
end
# Get logged HTTP headers from request.
def self.header_params(request)
headers = request.headers.to_h.select { |header, value| header.match?(/\AHTTP_/) }
headers = headers.transform_keys { |header| header.delete_prefix("HTTP_").downcase }
@@ -65,6 +86,7 @@ class DanbooruLogger
private_class_method
# @see https://docs.newrelic.com/docs/using-new-relic/data/customize-data/collect-custom-attributes/#ruby-att
def self.add_custom_attributes(attributes)
return unless defined?(::NewRelic)
::NewRelic::Agent.add_custom_attributes(attributes)

View File

@@ -1,3 +1,5 @@
# A DeviantArt API client.
#
# Authentication is via OAuth2 with the client credentials grant. Register a
# new app at https://www.deviantart.com/developers/ to obtain a client_id and
# client_secret. The app doesn't need to be published.

View File

@@ -1,3 +1,4 @@
# Builds an HTML diff between two pieces of text.
class DiffBuilder
attr_reader :this_text, :that_text, :pattern

View File

@@ -1,3 +1,6 @@
# A Discord API Client.
#
# @see https://discord.com/developers/docs/intro
class DiscordApiClient
extend Memoist

View File

@@ -1,3 +1,21 @@
# The parent class for a Discord slash command. Each slash command handled by
# Danbooru is a subclass of this class. Subclasses should set the {#name},
# {#description}, and {#options} class attributes defining the slash command's
# parameters, then implement the {#call} method to return a response to the
# command.
#
# The lifecycle of a Discord slash command is this:
#
# * First we register the command with Discord with an API call ({DiscordApiClient#register_slash_command}).
# * Then whenever someone uses the command, Discord sends us a HTTP request to
# https://danbooru.donmai.us/webhooks/receive.
# * We validate the request really came from Discord, then return the
# command's response.
#
# @abstract
# @see DiscordApiClient
# @see WebhooksController#receive
# @see https://discord.com/developers/docs/interactions/slash-commands
class DiscordSlashCommand
class WebhookVerificationError < StandardError; end
@@ -47,7 +65,7 @@ class DiscordSlashCommand
end
concerning :HelperMethods do
# The parameters passed to the command. A hash.
# @return [Hash] The parameters passed to the slash command by the Discord user.
def params
@params ||= data.dig(:data, :options).to_a.map do |opt|
[opt[:name], opt[:value]]

View File

@@ -1,14 +1,25 @@
require 'resolv'
# Validates that an email address is well-formed, is deliverable, and is not a
# disposable or throwaway email address. Also normalizes equivalent addresses to
# a single canonical form, so that users can't use different forms of the same
# address to register multiple accounts.
module EmailValidator
module_function
# https://www.regular-expressions.info/email.html
EMAIL_REGEX = /\A[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\z/
# Sites that ignore dots in email addresses, e.g. where `te.st@gmail.com` is
# the same as `test@gmail.com`.
IGNORE_DOTS = %w[gmail.com]
# Sites that allow plus addressing, e.g. `test+nospam@gmail.com`.
# @see https://en.wikipedia.org/wiki/Email_address#Subaddressing
IGNORE_PLUS_ADDRESSING = %w[gmail.com hotmail.com outlook.com live.com]
IGNORE_MINUS_ADDRESSING = %w[yahoo.com]
# Sites that have multiple domains mapping to the same logical email address.
CANONICAL_DOMAINS = {
"googlemail.com" => "gmail.com",
"hotmail.co.uk" => "outlook.com",
@@ -66,10 +77,16 @@ module EmailValidator
"gmx.us" => "gmx.net",
}
# Returns true if it's okay to connect to port 25. Disabled outside of
# production because many home ISPs blackhole port 25.
def smtp_enabled?
Rails.env.production?
end
# Normalize an email address by stripping out plus addressing and dots, if
# applicable, and rewriting the domain to a canonical domain.
# @param address [String] the email address to normalize
# @return [String] the normalized address
def normalize(address)
return nil unless address.count("@") == 1
@@ -83,10 +100,16 @@ module EmailValidator
"#{name}@#{domain}"
end
# Returns true if the email address is correctly formatted.
# @param [String] the email address
# @return [Boolean]
def is_valid?(address)
address.match?(EMAIL_REGEX)
end
# Returns true if the email is a throwaway or disposable email address.
# @param [String] the email address
# @return [Boolean]
def is_restricted?(address)
return false if Danbooru.config.email_domain_verification_list.blank?
@@ -96,6 +119,11 @@ module EmailValidator
true
end
# Returns true if the email can't be delivered. Checks if the domain has an MX
# record and responds to the RCPT TO command.
# @param to_address [String] the email address to check
# @param from_address [String] the email address to check from
# @return [Boolean]
def undeliverable?(to_address, from_address: Danbooru.config.contact_email, timeout: 3)
mail_server = mx_domain(to_address, timeout: timeout)
mail_server.nil? || rcpt_to_failed?(to_address, from_address, mail_server, timeout: timeout)
@@ -103,6 +131,13 @@ module EmailValidator
false
end
# Returns true if the email can't be delivered. Sends a RCPT TO command over
# port 25 to check if the mailbox exists.
# @param to_address [String] the email address to check
# @param from_address [String] the email address to check from
# @param mail_server [String] the DNS name of the SMTP server
# @param timeout [Integer] the network timeout
# @return [Boolean]
def rcpt_to_failed?(to_address, from_address, mail_server, timeout: nil)
return false unless smtp_enabled?
@@ -121,6 +156,11 @@ module EmailValidator
end
end
# Does a DNS MX record lookup of the domain in the email address and returns the
# name of the mail server, if it exists.
# @param to_address [String] the email address to check
# @param timeout [Integer] the network timeout
# @return [String] the DNS name of the mail server
def mx_domain(to_address, timeout: nil)
domain = Mail::Address.new(to_address).domain

View File

@@ -1,21 +1,48 @@
# Parse the output of the Postgres EXPLAIN command to get how many rows Postgres
# *thinks* a SQL query will return. This is an estimate, and only accurate for
# queries that have a single condition. If the query has multiple conditions,
# then Postgres will assume they're independent (have no overlap), which is not
# always the case.
#
# Used by {PostQueryBuilder#fast_count} to get a fast post count estimate for
# certain single metatag searches.
#
# ExplainParser.new(Post.all).query_plan
# => EXPLAIN (FORMAT JSON) SELECT "posts".* FROM "posts"
# => {
# "Node Type"=>"Seq Scan",
# "Parallel Aware"=>false,
# "Relation Name"=>"posts",
# "Alias"=>"posts",
# "Startup Cost"=>0.0,
# "Total Cost"=>780900.02,
# "Plan Rows"=>4413102,
# "Plan Width"=>1268
# }
#
# @see https://www.postgresql.org/docs/current/sql-explain.html
class ExplainParser
extend Memoist
attr_reader :relation
# @param the ActiveRecord relation
def initialize(relation)
@relation = relation
end
# @return [Hash] the Postgres query plan
def query_plan
result = ApplicationRecord.connection.select_one("EXPLAIN (FORMAT JSON) #{sql}")
json = JSON.parse(result["QUERY PLAN"])
json.first["Plan"]
end
# @return [Integer] the number of rows Postgres *thinks* the query will return
def row_count
query_plan["Plan Rows"]
end
# @return [String] the query's SQL
def sql
relation.reorder(nil).to_sql
end

View File

@@ -1,3 +1,4 @@
# Add a post to a forum topic.
class ForumUpdater
attr_reader :forum_topic

View File

@@ -1,13 +1,20 @@
# See also config/initializers/types.rb
# Ensure ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Inet is loaded in
# the dev environment when eager loading is disabled.
require "active_record/connection_adapters/postgresql_adapter"
# Define a custom IP address type for IP columns in the database. IP columns
# will be serialized and deserialized as a {Danbooru::IpAddress} object.
#
# @see config/initializers/types.rb
# @see https://www.bigbinary.com/blog/rails-5-attributes-api
class IpAddressType < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Inet
# @param value [String] the IP address from the database
def cast(value)
return nil if value.blank?
super(Danbooru::IpAddress.new(value))
end
# @param [Danbooru::IpAddress] the IP address object
def serialize(value)
value&.to_string
end

View File

@@ -1,5 +1,7 @@
# API client for https://ipregistry.co/
# An API client for https://ipregistry.co. Looks up IP address information,
# including the geolocation and whether the IP is a proxy.
#
# @see https://ipregistry.co/docs
class IpLookup
extend Memoist

View File

@@ -1,13 +1,21 @@
# An API client for Danbooru's internal IQDB instance. Can add images, remove
# images, and search for images in IQDB.
#
# @see https://github.com/danbooru/iqdb
class IqdbClient
class Error < StandardError; end
attr_reader :iqdb_url, :http
# Create a new IQDB API client.
# @param iqdb_url [String] the base URL of the IQDB server
# @param http [Danbooru::Http] the HTTP client to use
def initialize(iqdb_url: Danbooru.config.iqdb_url.to_s, http: Danbooru::Http.new)
@iqdb_url = iqdb_url.chomp("/")
@http = http
end
concerning :QueryMethods do
# Search for an image by file, URL, or post ID.
def search(post_id: nil, file: nil, url: nil, image_url: nil, file_url: nil, similarity: 0.0, high_similarity: 65.0, limit: 20)
limit = limit.to_i.clamp(1, 1000)
similarity = similarity.to_f.clamp(0.0, 100.0)
@@ -37,6 +45,10 @@ class IqdbClient
file.try(:close)
end
# Download an URL to a file.
# @param url [String] the URL to download
# @param type [Symbol] the type of URL to download (:preview_url or full :image_url)
# @return [MediaFile] the downloaded file
def download(url, type)
strategy = Sources::Strategies.find(url)
download_url = strategy.send(type)
@@ -44,8 +56,12 @@ class IqdbClient
file
end
def decorate_posts(json)
post_ids = json.map { |match| match["post_id"] }
# Transform the JSON returned by IQDB to add the full post data for each
# match.
# @param matches [Array<Hash>] the array of IQDB matches
# @return [Array<Hash>] the array of IQDB matches, with post data
def decorate_posts(matches)
post_ids = matches.map { |match| match["post_id"] }
posts = Post.where(id: post_ids).group_by(&:id).transform_values(&:first)
json.map do |match|
@@ -55,6 +71,8 @@ class IqdbClient
end
end
# Add a post to IQDB.
# @param post [Post] the post to add
def add_post(post)
return unless post.has_preview?
preview_file = post.file(:preview)
@@ -62,20 +80,31 @@ class IqdbClient
end
concerning :HttpMethods do
# Search for an image in IQDB.
# @param file [File] the image to search
def query(file, limit: 20)
file = HTTP::FormData::File.new(file)
request(:post, "query", form: { file: file }, params: { limit: limit })
end
# Add a post to IQDB.
# @param post_id [Integer] the post to add
# @param file [File] the image to add
def add(post_id, file)
file = HTTP::FormData::File.new(file)
request(:post, "images/#{post_id}", form: { file: file })
end
# Remove an image from IQDB.
# @param post_id [Integer] the post to remove
def remove(post_id)
request(:delete, "images/#{post_id}")
end
# Send a request to IQDB.
# @param method [String] the HTTP method
# @param url [String] the IQDB url
# @param options [Hash] the URL params to send
def request(method, url, **options)
return [] if iqdb_url.blank? # do nothing if iqdb isn't configured
response = http.timeout(30).send(method, "#{iqdb_url}/#{url}", **options)

View File

@@ -1,3 +1,6 @@
# An API client for Mastodon.
#
# @see https://docs.joinmastodon.org/api
class MastodonApiClient
extend Memoist
attr_reader :json

View File

@@ -1,3 +1,4 @@
# An API client for the NicoSeiga XML API.
class NicoSeigaApiClient
extend Memoist
XML_API = "https://seiga.nicovideo.jp/api"

View File

@@ -1,3 +1,5 @@
# Sanitizes the HTML used in notes. Only safe HTML tags, HTML attributes, and
# CSS properties are allowed.
module NoteSanitizer
ALLOWED_ELEMENTS = %w[
code center tn h1 h2 h3 h4 h5 h6 a span div blockquote br p ul li ol em
@@ -50,6 +52,9 @@ module NoteSanitizer
vertical-align
]
# Sanitize a string of HTML.
# @param text [String] the HTML to sanitize
# @return [String] the sanitized HTML
def self.sanitize(text)
text.gsub!(/<( |-|3|:|>|\Z)/, "&lt;\\1")
@@ -76,6 +81,8 @@ module NoteSanitizer
)
end
# Convert absolute Danbooru links inside notes to relative links.
# https://danbooru.donmai.us/posts/1 is converted to /posts/1.
def self.relativize_links(node:, **env)
return unless node.name == "a" && node["href"].present?

View File

@@ -1,4 +1,6 @@
# Notes on Pixiv's AJAX API:
# A client for Pixiv's AJAX API.
#
# Notes on the API:
#
# * The user agent must be spoofed as a browser user agent, otherwise the API
# will return an error.
@@ -359,23 +361,40 @@ class PixivAjaxClient
attr_reader :phpsessid, :http
# @param phpsessid [String] the Pixiv login cookie
# @param http [Danbooru::Http] the HTTP client to use for Pixiv
def initialize(phpsessid, http: Danbooru::Http.new)
@phpsessid = phpsessid
@http = http
end
# Return the illust data for a Pixiv illustration.
# @see https://www.pixiv.net/ajax/illust/87598468
# @param illust_id [Integer] the Pixiv illustration id
# @return [Hash] the illustration data
def illust(illust_id)
get("https://www.pixiv.net/ajax/illust/#{illust_id}").with_indifferent_access
end
# Return the data for a Pixiv manga illust with multiple pages.
# @see https://www.pixiv.net/ajax/illust/87598468/pages
# @param illust_id [Integer] the Pixiv illustration id
# @return [Hash] the illustration data
def pages(illust_id)
get("https://www.pixiv.net/ajax/illust/#{illust_id}/pages")
end
# Return the data for a Pixiv ugoira.
# @see https://www.pixiv.net/ajax/illust/74932152/ugoira_meta
# @param illust_id [Integer] the Pixiv illustration id
# @return [Hash] the ugoira data
def ugoira_meta(illust_id)
get("https://www.pixiv.net/ajax/illust/#{illust_id}/ugoira_meta").with_indifferent_access
end
# Perform a GET request for a Pixiv URL.
# @param url [String] the Pixiv URL
# @return [Hash] the parsed response, or blank on error
def get(url)
response = client.cache(1.minute).get(url)
@@ -387,6 +406,7 @@ class PixivAjaxClient
end
end
# @return [Danbooru::Http] the HTTP client used for Pixiv
def client
@client ||= http.headers("User-Agent": USER_AGENT).cookies(PHPSESSID: phpsessid)
end

View File

@@ -1,3 +1,6 @@
# Posts new appeals to the Deletion appeal thread once per hour.
#
# @see DanbooruMaintenance#hourly
module PostAppealForumUpdater
APPEAL_TOPIC_TITLE = "Deletion appeal thread"

View File

@@ -1,3 +1,4 @@
# Delete posts that were unapproved after three days.
module PostPruner
module_function

View File

@@ -1,3 +1,11 @@
# Define a custom validator for tag names. Tags must be plain ASCII, no spaces,
# no redundant underscores, no conflicts with metatags, and can't begin with
# certain special characters.
#
# @example
# validates :name, tag_name: true
#
# @see https://guides.rubyonrails.org/active_record_validations.html#custom-validators
class TagNameValidator < ActiveModel::EachValidator
MAX_TAG_LENGTH = 170

View File

@@ -1,3 +1,8 @@
# Removes tag aliases and implications if they haven't had any new uploads in
# the last two years. Runs weekly. Posts a message to the forum when aliases or
# implications are retired.
#
# @see DanbooruMaintenance#weekly
module TagRelationshipRetirementService
module_function

View File

@@ -1,3 +1,9 @@
# Define a custom validator for user names.
#
# @example
# validates :name, user_name: true
#
# @see https://guides.rubyonrails.org/active_record_validations.html#custom-validators
class UserNameValidator < ActiveModel::EachValidator
def validate_each(rec, attr, value)
name = value