docs: add documentation for various classes in app/logical.
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
# Parse an PNG and return whether or not it's animated.
|
||||
class APNGInspector
|
||||
attr_reader :frames
|
||||
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# https://github.com/plataformatec/responders
|
||||
# https://github.com/plataformatec/responders/blob/master/lib/action_controller/responder.rb
|
||||
# Hooks into `respond_with` to add some custom behavior, including support for
|
||||
# the `expires_in`, `expiry`, and `only` params.
|
||||
#
|
||||
# @see https://github.com/plataformatec/responders
|
||||
# @see https://github.com/plataformatec/responders/blob/master/lib/action_controller/responder.rb
|
||||
class ApplicationResponder < ActionController::Responder
|
||||
# this is called by respond_with for non-html, non-js responses.
|
||||
def to_format
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
# Demote all approvers who haven't approved at least 30 posts in the last 45
|
||||
# days. Moderators and recently promoted approvers are exempt. Runs as a monthly
|
||||
# maintenance task. Approvers who are facing demotion are sent a weekly warning
|
||||
# dmail first.
|
||||
#
|
||||
# @see DanbooruMaintenance#monthly
|
||||
module ApproverPruner
|
||||
module_function
|
||||
|
||||
APPROVAL_PERIOD = 45.days
|
||||
MINIMUM_APPROVALS = 30
|
||||
|
||||
# Get the list of inactive approvers.
|
||||
# @return [Array<User>] the list of inactive approvers
|
||||
def inactive_approvers
|
||||
approvers = User.bit_prefs_match(:can_approve_posts, true)
|
||||
approvers = approvers.where("level < ?", User::Levels::MODERATOR)
|
||||
@@ -16,9 +24,10 @@ module ApproverPruner
|
||||
end
|
||||
end
|
||||
|
||||
# Demote all inactive approvers
|
||||
def prune!
|
||||
inactive_approvers.each do |user|
|
||||
CurrentUser.scoped(User.system, "127.0.0.1") do
|
||||
CurrentUser.scoped(User.system) do
|
||||
user.update!(can_approve_posts: false)
|
||||
user.feedback.create(category: "neutral", body: "Lost approval privileges", creator: User.system)
|
||||
|
||||
@@ -31,6 +40,7 @@ module ApproverPruner
|
||||
end
|
||||
end
|
||||
|
||||
# Send a warning dmail to approvers who are pending demotion.
|
||||
def dmail_inactive_approvers!
|
||||
days_until_next_month = (Date.current.next_month.beginning_of_month - Date.current).to_i
|
||||
return unless days_until_next_month <= 21
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# Find the artist entry for a given artist profile URL.
|
||||
module ArtistFinder
|
||||
module_function
|
||||
|
||||
@@ -131,6 +132,14 @@ module ArtistFinder
|
||||
%r{\Ahttps?://(?:[a-zA-Z0-9_-]+\.)*#{domain}/\z}i
|
||||
end)
|
||||
|
||||
# Find the artist for a given artist profile URL. May return multiple Artists
|
||||
# in the event of duplicate artist entries.
|
||||
#
|
||||
# Uses a path-stripping algorithm to find any artist URL that is a prefix
|
||||
# of the given URL. A site blacklist is used to prevent false positives.
|
||||
#
|
||||
# @param url [String] the artist profile URL
|
||||
# @return [Array<Artist>] the list of matching artists
|
||||
def find_artists(url)
|
||||
url = ArtistUrl.normalize(url)
|
||||
artists = []
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
# Autocomplete tags, usernames, pools, and more.
|
||||
#
|
||||
# @example
|
||||
# AutocompleteService.new("touho", :tag).autocomplete_results
|
||||
# #=> [{ type: :tag, label: "touhou", value: "touhou", category: 3, post_count: 42 }]
|
||||
#
|
||||
# @see AutocompleteController
|
||||
class AutocompleteService
|
||||
extend Memoist
|
||||
|
||||
@@ -18,6 +25,11 @@ class AutocompleteService
|
||||
|
||||
attr_reader :query, :type, :limit, :current_user
|
||||
|
||||
# Perform completion for the given search type and query.
|
||||
# @param query [String] the string being completed
|
||||
# @param type [String] the type of completion being performed
|
||||
# @param current_user [User] the user we're performing completion for
|
||||
# @param limit [Integer] the max number of results to return
|
||||
def initialize(query, type, current_user: User.anonymous, limit: 10)
|
||||
@query = query.to_s
|
||||
@type = type.to_s.to_sym
|
||||
@@ -25,6 +37,8 @@ class AutocompleteService
|
||||
@limit = limit
|
||||
end
|
||||
|
||||
# Return the results of the completion.
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_results
|
||||
case type
|
||||
when :tag_query
|
||||
@@ -52,6 +66,9 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a tag search (a regular tag or a metatag)
|
||||
# @param string [String] the string to complete
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_tag_query(string)
|
||||
term = PostQueryBuilder.new(string).terms.first
|
||||
return [] if term.nil?
|
||||
@@ -64,10 +81,20 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Find tags matching a search.
|
||||
#
|
||||
# If the string is non-English, translate it to a Danbooru tag.
|
||||
# If the string is a slash abbreviation, expand the abbreviation.
|
||||
# If the string has a wildcard, do a wildcard search.
|
||||
# If the string doesn't match anything, perform autocorrect.
|
||||
#
|
||||
# @param string [String] the string to complete
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_tag(string)
|
||||
return [] if string.size > TagNameValidator::MAX_TAG_LENGTH
|
||||
return [] if string.start_with?("http://", "https://")
|
||||
|
||||
# XXX convert to NFKC? deaccent?
|
||||
if !string.ascii_only?
|
||||
results = tag_other_name_matches(string)
|
||||
elsif string.starts_with?("/")
|
||||
@@ -90,6 +117,9 @@ class AutocompleteService
|
||||
results
|
||||
end
|
||||
|
||||
# Find tags or tag aliases matching a wildcard search.
|
||||
# @param string [String] the string to complete
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def tag_matches(string)
|
||||
name_matches = Tag.nonempty.name_matches(string).order(post_count: :desc).limit(limit)
|
||||
alias_matches = Tag.nonempty.alias_matches(string).order(post_count: :desc).limit(limit)
|
||||
@@ -103,6 +133,12 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Find tags matching a slash abbreviation.
|
||||
# Example: /evth => eyebrows_visible_through_hair
|
||||
#
|
||||
# @param string [String] the string to complete
|
||||
# @param max_length [Integer] the max abbreviation length
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def tag_abbreviation_matches(string, max_length: 10)
|
||||
return [] if string.size > max_length
|
||||
|
||||
@@ -113,6 +149,11 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Find tags matching a mispelled tag.
|
||||
# Example: logn_hair => long_hair
|
||||
#
|
||||
# @param string [String] the string to complete
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def tag_autocorrect_matches(string)
|
||||
# autocorrect uses trigram indexing, which needs at least 3 alphanumeric characters to work.
|
||||
return [] if string.remove(/[^a-zA-Z0-9]/).size < 3
|
||||
@@ -124,6 +165,12 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Find tags matching a non-English string. Does a `name*` search in wiki page
|
||||
# and artist other names to translate the non-English tag to a Danbooru tag.
|
||||
# Example: 東方 => touhou.
|
||||
#
|
||||
# @param string [String] the string to complete
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def tag_other_name_matches(string)
|
||||
artists = Artist.undeleted.where_any_in_array_starts_with(:other_names, string)
|
||||
wikis = WikiPage.undeleted.where_any_in_array_starts_with(:other_names, string)
|
||||
@@ -137,6 +184,10 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a metatag.
|
||||
# @param metatag [String] the type of metatag to complete
|
||||
# @param value [String] the value of the metatag
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_metatag(metatag, value)
|
||||
results = case metatag.to_sym
|
||||
when :user, :approver, :commenter, :comm, :noter, :noteupdater, :commentaryupdater,
|
||||
@@ -159,6 +210,10 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a static metatag: rating, filetype, etc.
|
||||
# @param metatag [String] the type of metatag to complete
|
||||
# @param value [String] the value of the metatag
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_static_metatag(metatag, value)
|
||||
values = STATIC_METATAGS[metatag.to_sym]
|
||||
results = values.select { |v| v.starts_with?(value.downcase) }.sort.take(limit)
|
||||
@@ -168,6 +223,9 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a pool name. Does a `*name*` search.
|
||||
# @param string [String] the name of the pool
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_pool(string)
|
||||
string = "*" + string + "*" unless string.include?("*")
|
||||
pools = Pool.undeleted.name_matches(string).search(order: "post_count").limit(limit)
|
||||
@@ -177,6 +235,9 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a favorite group name. Does a `*name*` search.
|
||||
# @param string [String] the name of the favgroup
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_favorite_group(string)
|
||||
string = "*" + string + "*" unless string.include?("*")
|
||||
favgroups = FavoriteGroup.visible(current_user).where(creator: current_user).name_matches(string).search(order: "post_count").limit(limit)
|
||||
@@ -186,6 +247,9 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a saved search label. Does a `*name*` search.
|
||||
# @param string [String] the name of the label
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_saved_search_label(string)
|
||||
string = "*" + string + "*" unless string.include?("*")
|
||||
labels = current_user.saved_searches.labels_like(string).take(limit)
|
||||
@@ -195,6 +259,9 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete an artist name. Does a `name*` search.
|
||||
# @param string [String] the name of the artist
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_artist(string)
|
||||
string = string + "*" unless string.include?("*")
|
||||
artists = Artist.undeleted.name_matches(string).search(order: "post_count").limit(limit)
|
||||
@@ -204,6 +271,9 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a wiki name. Does a `name*` search.
|
||||
# @param string [String] the name of the wiki
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_wiki_page(string)
|
||||
string = string + "*" unless string.include?("*")
|
||||
wiki_pages = WikiPage.undeleted.title_matches(string).search(order: "post_count").limit(limit)
|
||||
@@ -213,6 +283,9 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a user name. Does a `name*` search.
|
||||
# @param string [String] the name of the user
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_user(string)
|
||||
string = string + "*" unless string.include?("*")
|
||||
users = User.search(name_matches: string, current_user_first: true, order: "post_upload_count").limit(limit)
|
||||
@@ -222,17 +295,27 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Complete an @mention for a user name. Does a `name*` search.
|
||||
# @param string [String] the name of the user
|
||||
# @return [Array<Hash>] the autocomplete results
|
||||
def autocomplete_mention(string)
|
||||
autocomplete_user(string).map do |result|
|
||||
{ **result, value: "@" + result[:value] }
|
||||
end
|
||||
end
|
||||
|
||||
# Complete a search typed in the browser address bar.
|
||||
# @param string [String] the name of the tag
|
||||
# @return [Array<(String, [Array<String>])>] the autocomplete results
|
||||
# @see https://en.wikipedia.org/wiki/OpenSearch
|
||||
# @see https://developer.mozilla.org/en-US/docs/Web/OpenSearch
|
||||
def autocomplete_opensearch(string)
|
||||
results = autocomplete_tag(string).map { |result| result[:value] }
|
||||
[query, results]
|
||||
end
|
||||
|
||||
# How long autocomplete results can be cached. Cache short result lists (<10
|
||||
# results) for less time because they're more likely to change.
|
||||
def cache_duration
|
||||
if autocomplete_results.size == limit
|
||||
24.hours
|
||||
@@ -241,6 +324,7 @@ class AutocompleteService
|
||||
end
|
||||
end
|
||||
|
||||
# Whether the results can be safely cached with `Cache-Control: public`.
|
||||
# Queries that don't depend on the current user are safe to cache publicly.
|
||||
def cache_publicly?
|
||||
if type == :tag_query && parsed_search&.type == :tag
|
||||
|
||||
@@ -1,22 +1,40 @@
|
||||
# Export all public data in a model to BigQuery and to Google Cloud Storage.
|
||||
|
||||
# Perform a daily database dump to BigQuery and to Google Cloud Storage. This
|
||||
# contains all data visible to anonymous users.
|
||||
#
|
||||
# The database dumps are publicly accessible. The BigQuery data is at
|
||||
# `danbooru1.danbooru_public.{table}`. The Google Cloud Storage data is at
|
||||
# `gs://danbooru_public/data/{table}.json`. The storage bucket contains the data
|
||||
# in newline-delimited JSON format.
|
||||
#
|
||||
# @see DanbooruMaintenance#daily
|
||||
# @see https://console.cloud.google.com/storage/browser/danbooru_public
|
||||
# @see https://console.cloud.google.com/bigquery?d=danbooru_public&p=danbooru1&t=posts&page=table
|
||||
# @see https://cloud.google.com/bigquery/docs
|
||||
# @see https://cloud.google.com/storage/docs
|
||||
# @see https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON
|
||||
class BigqueryExportService
|
||||
extend Memoist
|
||||
|
||||
attr_reader :model, :dataset_name, :credentials
|
||||
|
||||
# Prepare to dump a table. Call {#export!} to dump it.
|
||||
# @param model [ApplicationRecord] the database table to dump
|
||||
# @param dataset_name [String] the BigQuery dataset name
|
||||
# @param credentials [String] the Google Cloud credentials (in JSON format)
|
||||
def initialize(model = nil, dataset_name: "danbooru_public", credentials: default_credentials)
|
||||
@model = model
|
||||
@dataset_name = dataset_name
|
||||
@credentials = credentials
|
||||
end
|
||||
|
||||
# Start a background job for each table to export it to BigQuery.
|
||||
def self.async_export_all!(**options)
|
||||
models.each do |model|
|
||||
BigqueryExportJob.perform_later(model: model, **options)
|
||||
end
|
||||
end
|
||||
|
||||
# The list of database tables to dump.
|
||||
def self.models
|
||||
Rails.application.eager_load!
|
||||
|
||||
@@ -29,6 +47,7 @@ class BigqueryExportService
|
||||
credentials.present?
|
||||
end
|
||||
|
||||
# Dump the table to Cloud Storage and BigQuery.
|
||||
def export!
|
||||
return unless enabled? && records.any?
|
||||
|
||||
@@ -36,7 +55,7 @@ class BigqueryExportService
|
||||
upload_to_bigquery!(file)
|
||||
end
|
||||
|
||||
# Dump the model records to a gzipped, newline-delimited JSON tempfile.
|
||||
# Dump the table's records to a gzipped, newline-delimited JSON tempfile.
|
||||
def dump_records!
|
||||
file = Tempfile.new("danbooru-export-dump-", binmode: true)
|
||||
file = Zlib::GzipWriter.new(file)
|
||||
@@ -51,8 +70,7 @@ class BigqueryExportService
|
||||
file
|
||||
end
|
||||
|
||||
# GCS: gs://danbooru_public/data/{model}.json
|
||||
# BQ: danbooru1.danbooru_public.{model}
|
||||
# Upload the JSON dump to Cloud Storage, then load it into BigQuery.
|
||||
def upload_to_bigquery!(file)
|
||||
table_name = model.model_name.collection
|
||||
gsfilename = "data/#{table_name}.json"
|
||||
@@ -64,24 +82,27 @@ class BigqueryExportService
|
||||
job
|
||||
end
|
||||
|
||||
# private
|
||||
|
||||
# The list of records to dump.
|
||||
def records
|
||||
model.visible(User.anonymous)
|
||||
end
|
||||
|
||||
# Find or create the BigQuery dataset.
|
||||
def dataset
|
||||
bigquery.dataset(dataset_name) || bigquery.create_dataset(dataset_name)
|
||||
end
|
||||
|
||||
# Find or create the Google Storage bucket.
|
||||
def bucket
|
||||
storage.bucket(dataset_name) || storage.create_bucket(dataset_name, acl: "public", default_acl: "public", storage_class: "standard", location: "us-east1")
|
||||
end
|
||||
|
||||
# The BigQuery API client.
|
||||
def bigquery
|
||||
Google::Cloud::Bigquery.new(credentials: credentials)
|
||||
end
|
||||
|
||||
# The Cloud Storage API client.
|
||||
def storage
|
||||
Google::Cloud::Storage.new(credentials: credentials)
|
||||
end
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# Process a bulk update request. Parses the request and applies each line in
|
||||
# sequence.
|
||||
class BulkUpdateRequestProcessor
|
||||
# Maximum tag size allowed by the rename command before an alias must be used.
|
||||
MAXIMUM_RENAME_COUNT = 200
|
||||
@@ -17,6 +19,7 @@ class BulkUpdateRequestProcessor
|
||||
validate :validate_script
|
||||
validate :validate_script_length
|
||||
|
||||
# @param bulk_update_request [String] the BUR
|
||||
def initialize(bulk_update_request)
|
||||
@bulk_update_request = bulk_update_request
|
||||
end
|
||||
@@ -50,6 +53,8 @@ class BulkUpdateRequestProcessor
|
||||
end
|
||||
end
|
||||
|
||||
# Validate the bulk update request when it is created or approved.
|
||||
#
|
||||
# validation_context will be either :request (when the BUR is first created
|
||||
# or edited) or :approval (when the BUR is approved). Certain validations
|
||||
# only run when the BUR is requested, not when it's approved.
|
||||
@@ -117,12 +122,15 @@ class BulkUpdateRequestProcessor
|
||||
end
|
||||
end
|
||||
|
||||
# Validate that the script isn't too long.
|
||||
def validate_script_length
|
||||
if commands.size > MAXIMUM_SCRIPT_LENGTH
|
||||
errors.add(:base, "Bulk update request is too long (maximum size: #{MAXIMUM_SCRIPT_LENGTH} lines). Split your request into smaller chunks and try again.")
|
||||
end
|
||||
end
|
||||
|
||||
# Apply the script.
|
||||
# @param approver [User] the approver of the request
|
||||
def process!(approver)
|
||||
ActiveRecord::Base.transaction do
|
||||
commands.map do |command, *args|
|
||||
@@ -162,6 +170,8 @@ class BulkUpdateRequestProcessor
|
||||
end
|
||||
end
|
||||
|
||||
# The list of tags in the script. Used for search BURs by tag.
|
||||
# @return [Tag] the list of tags
|
||||
def affected_tags
|
||||
commands.flat_map do |command, *args|
|
||||
case command
|
||||
@@ -178,6 +188,7 @@ class BulkUpdateRequestProcessor
|
||||
end.sort.uniq
|
||||
end
|
||||
|
||||
# Returns true if a non-Admin is allowed to approve a rename or alias request.
|
||||
def is_tag_move_allowed?
|
||||
commands.all? do |command, *args|
|
||||
case command
|
||||
@@ -194,6 +205,8 @@ class BulkUpdateRequestProcessor
|
||||
end
|
||||
end
|
||||
|
||||
# Convert the BUR to DText format.
|
||||
# @return [String]
|
||||
def to_dtext
|
||||
commands.map do |command, *args|
|
||||
case command
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# Rejects bulk update requests that haven't been approved in 60 days.
|
||||
module BulkUpdateRequestPruner
|
||||
module_function
|
||||
|
||||
# Posts a warning when a bulk update request is pending automatic rejection in 5 days.
|
||||
def warn_old
|
||||
BulkUpdateRequest.old.pending.find_each do |bulk_update_request|
|
||||
if bulk_update_request.forum_topic
|
||||
@@ -12,6 +14,7 @@ module BulkUpdateRequestPruner
|
||||
end
|
||||
end
|
||||
|
||||
# Rejects bulk update requests that haven't been approved in 60 days.
|
||||
def reject_expired
|
||||
BulkUpdateRequest.expired.pending.find_each do |bulk_update_request|
|
||||
ApplicationRecord.transaction do
|
||||
|
||||
@@ -1,4 +1,17 @@
|
||||
# A wrapper around `Rails.cache` that adds some extra helper methods. All
|
||||
# Danbooru code should use this class instead of using `Rails.cache` directly.
|
||||
#
|
||||
# In production, the cache is backed by Redis. In development, it's a temporary
|
||||
# in-memory cache.
|
||||
#
|
||||
# @see config/initializers/cache_store.rb
|
||||
# @see https://guides.rubyonrails.org/caching_with_rails.html#cache-stores
|
||||
# @see https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html
|
||||
class Cache
|
||||
# Get multiple values from the cache at once.
|
||||
# @param keys [Array<String>] the list of keys to fetch
|
||||
# @param prefix [String] a prefix for each cache key
|
||||
# @return [Hash<String, Object>] a map from cache keys to cache values
|
||||
def self.get_multi(keys, prefix)
|
||||
sanitized_key_to_key_hash = keys.map do |key|
|
||||
["#{prefix}:#{Cache.hash(key)}", key]
|
||||
@@ -14,28 +27,41 @@ class Cache
|
||||
keys_to_values_hash
|
||||
end
|
||||
|
||||
# Get a value from the cache. If the value isn't in the cache, use the block
|
||||
# to generate the value.
|
||||
# @param key [String] the key to fetch
|
||||
# @param expiry_in_seconds [Integer] the lifetime of the cached value
|
||||
# @param options [Hash] options to pass to Rails.cache.fetch
|
||||
# @return the cached value
|
||||
def self.get(key, expiry_in_seconds = nil, **options, &block)
|
||||
Rails.cache.fetch(key, expires_in: expiry_in_seconds, **options, &block)
|
||||
end
|
||||
|
||||
# Write a value to the cache.
|
||||
# @param key [String] the key to cache
|
||||
# @param value [Object] the value to cache
|
||||
# @param expiry_in_seconds [Integer] the lifetime of the cached value
|
||||
# @return the cached value
|
||||
def self.put(key, value, expiry_in_seconds = nil)
|
||||
Rails.cache.write(key, value, expires_in: expiry_in_seconds)
|
||||
value
|
||||
end
|
||||
|
||||
# Remove a value from the cache.
|
||||
# @param key [String] the key to remove
|
||||
def self.delete(key)
|
||||
Rails.cache.delete(key)
|
||||
nil
|
||||
end
|
||||
|
||||
# Clear the entire cache.
|
||||
def self.clear
|
||||
Rails.cache.clear
|
||||
end
|
||||
|
||||
def self.sanitize(key)
|
||||
key.gsub(/\W/) {|x| "%#{x.ord}"}.slice(0, 230)
|
||||
end
|
||||
|
||||
# Hash a cache key.
|
||||
# @param string [String] the cache key to hash
|
||||
# @return [String] the hashed key
|
||||
def self.hash(string)
|
||||
Digest::SHA256.base64digest(string)
|
||||
end
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# A simple Cloudflare API client for purging cached images after they're
|
||||
# regenerated or deleted.
|
||||
class CloudflareService
|
||||
attr_reader :api_token, :zone
|
||||
|
||||
@@ -9,6 +11,9 @@ class CloudflareService
|
||||
api_token.present? && zone.present?
|
||||
end
|
||||
|
||||
# Purge a list of URLs from Cloudflare's cache.
|
||||
# @param urls [Array<String>] the list of URLs to purge
|
||||
# @see https://api.cloudflare.com/#zone-purge-files-by-url
|
||||
def purge_cache(urls)
|
||||
return unless enabled?
|
||||
|
||||
|
||||
@@ -1,3 +1,18 @@
|
||||
# Generates a set of CSS variables for site's color palette.
|
||||
#
|
||||
# The CSS variables are named like `var(--{hue}-{i})`, where `hue` is the color
|
||||
# name (see below) and `i` is the brightless level (0-9, 0 is white and 9 is
|
||||
# black).
|
||||
#
|
||||
# Based on the HSLuv color space. The color palette is designed to be
|
||||
# perceptually uniform, that is, to have equal brightness levels across each
|
||||
# color.
|
||||
#
|
||||
# @see https://www.hsluv.org/
|
||||
# @see https://en.wikipedia.org/wiki/HSLuv
|
||||
# @see https://github.com/hsluv/hsluv
|
||||
# @see https://danbooru.donmai.us/static/colors
|
||||
# @see app/javascript/src/styles/base/040_colors.css
|
||||
module ColorPalette
|
||||
module_function
|
||||
|
||||
|
||||
@@ -1,3 +1,15 @@
|
||||
# A global variable containing the current user, the current IP address, the
|
||||
# current user's country code, and whether safe mode is enabled.
|
||||
#
|
||||
# The current user is set during a request by {ApplicationController#set_current_user},
|
||||
# which calls {SessionLoader#load}. The current user will not be set outside of
|
||||
# the request cycle, for example, during background jobs, cron jobs, or on the
|
||||
# console. For this reason, code outside of controllers and views, such as
|
||||
# models, shouldn't rely on or assume the current user exists.
|
||||
#
|
||||
# @see https://api.rubyonrails.org/classes/ActiveSupport/CurrentAttributes.html
|
||||
# @see ApplicationController#set_current_user
|
||||
# @see SessionLoader#load
|
||||
class CurrentUser < ActiveSupport::CurrentAttributes
|
||||
attribute :user, :ip_addr, :country, :safe_mode
|
||||
|
||||
@@ -5,6 +17,10 @@ class CurrentUser < ActiveSupport::CurrentAttributes
|
||||
delegate :id, to: :user, allow_nil: true
|
||||
delegate_missing_to :user
|
||||
|
||||
# Run a block of code as another user.
|
||||
# @param user [User] the user to run as
|
||||
# @param ip_addr [String] the IP address to run as
|
||||
# @yield the block
|
||||
def self.scoped(user, ip_addr = "127.0.0.1", &block)
|
||||
set(user: user, ip_addr: ip_addr) do
|
||||
yield user
|
||||
|
||||
@@ -1,20 +1,38 @@
|
||||
require 'cgi'
|
||||
require 'uri'
|
||||
|
||||
# The DText class handles Danbooru's markup language, DText. Parsing DText is
|
||||
# handled by the DTextRagel class in the dtext_rb gem.
|
||||
#
|
||||
# @see https://github.com/evazion/dtext_rb
|
||||
# @see https://danbooru.donmai.us/wiki_pages/help:dtext
|
||||
class DText
|
||||
MENTION_REGEXP = /(?<=^| )@\S+/
|
||||
|
||||
def self.format_text(text, data: nil, **options)
|
||||
# Convert a string of DText to HTML.
|
||||
# @param text [String] The DText input
|
||||
# @param inline [Boolean] if true, allow only inline constructs. Ignore
|
||||
# block-level constructs, such as paragraphs, quotes, lists, and tables.
|
||||
# @param disable_mentions [Boolean] if true, don't generate @mentions.
|
||||
# @param base_url [String, nil] if present, convert relative URLs to absolute URLs.
|
||||
# @param data cached wiki/tag/artist data generated by {#preprocess}.
|
||||
# @return [String, nil] The HTML output
|
||||
def self.format_text(text, data: nil, inline: false, disable_mentions: false, base_url: nil)
|
||||
return nil if text.nil?
|
||||
data = preprocess([text]) if data.nil?
|
||||
text = parse_embedded_tag_request(text)
|
||||
html = DTextRagel.parse(text, **options)
|
||||
html = DTextRagel.parse(text, inline: inline, disable_mentions: disable_mentions, base_url: base_url)
|
||||
html = postprocess(html, *data)
|
||||
html
|
||||
rescue DTextRagel::Error
|
||||
""
|
||||
end
|
||||
|
||||
# Preprocess a set of DText messages and collect all tag, artist, and wiki
|
||||
# page references. Called before rendering a collection of DText messages
|
||||
# (e.g. comments or forum posts) to do all database lookups in one batch.
|
||||
# @param [Array<String>] a list of DText strings
|
||||
# @return an array of wiki pages, tags, and artists
|
||||
def self.preprocess(dtext_messages)
|
||||
dtext_messages = dtext_messages.map { |message| parse_embedded_tag_request(message) }
|
||||
names = dtext_messages.map { |message| parse_wiki_titles(message) }.flatten.uniq
|
||||
@@ -25,6 +43,11 @@ class DText
|
||||
[wiki_pages, tags, artists]
|
||||
end
|
||||
|
||||
# Rewrite the HTML produced by {#format_text} to colorize wiki links.
|
||||
# @param wiki_pages [Array<WikiPage>]
|
||||
# @param tags [Array<Tag>]
|
||||
# @param artists [Array<Artist>]
|
||||
# @return [String] the HTML output
|
||||
def self.postprocess(html, wiki_pages, tags, artists)
|
||||
fragment = Nokogiri::HTML.fragment(html)
|
||||
|
||||
@@ -69,11 +92,18 @@ class DText
|
||||
fragment.to_s
|
||||
end
|
||||
|
||||
# Wrap a DText message in a [quote] block.
|
||||
# @param message [String] the DText to quote
|
||||
# @param creator_name [String] the name of the user to quote.
|
||||
# @return [String] the quoted DText
|
||||
def self.quote(message, creator_name)
|
||||
stripped_body = DText.strip_blocks(message, "quote")
|
||||
"[quote]\n#{creator_name} said:\n\n#{stripped_body}\n[/quote]\n\n"
|
||||
end
|
||||
|
||||
# Convert `[bur:<id>]`, `[ta:<id>]`, `[ti:<id>]` tags to DText.
|
||||
# @param text [String] the DText input
|
||||
# @return [String] the DText output
|
||||
def self.parse_embedded_tag_request(text)
|
||||
text = parse_embedded_tag_request_type(text, TagAlias, /\[ta:(?<id>\d+)\]/m)
|
||||
text = parse_embedded_tag_request_type(text, TagImplication, /\[ti:(?<id>\d+)\]/m)
|
||||
@@ -81,6 +111,11 @@ class DText
|
||||
text
|
||||
end
|
||||
|
||||
# Convert a `[bur:<id>]`, `[ta:<id>]`, or `[ti:<id>]` tag to DText.
|
||||
# @param text [String] the DText input
|
||||
# @param tag_request [BulkUpdateRequest, TagAlias, TagImplication]
|
||||
# @param pattern [Regexp]
|
||||
# @return [String] the DText output
|
||||
def self.parse_embedded_tag_request_type(text, tag_request, pattern)
|
||||
text.gsub(pattern) do |match|
|
||||
obj = tag_request.find_by_id($~[:id])
|
||||
@@ -88,6 +123,9 @@ class DText
|
||||
end
|
||||
end
|
||||
|
||||
# Convert a `[bur:<id>]`, `[ta:<id>]`, or `[ti:<id>]` tag to DText.
|
||||
# @param obj [BulkUpdateRequest, TagAlias, TagImplication] the object to convert
|
||||
# @return [String] the DText output
|
||||
def self.tag_request_message(obj)
|
||||
if obj.is_a?(TagRelationship)
|
||||
if obj.is_active?
|
||||
@@ -118,6 +156,9 @@ class DText
|
||||
end
|
||||
end
|
||||
|
||||
# Return a list of user names mentioned in a string of DText. Ignore mentions in [quote] blocks.
|
||||
# @param text [String] the string of DText
|
||||
# @return [Array<String>] the list of user names
|
||||
def self.parse_mentions(text)
|
||||
text = strip_blocks(text.to_s, "quote")
|
||||
|
||||
@@ -128,6 +169,9 @@ class DText
|
||||
names.uniq
|
||||
end
|
||||
|
||||
# Return a list of wiki pages mentioned in a string of DText.
|
||||
# @param text [String] the string of DText
|
||||
# @return [Array<String>] the list of wiki page names
|
||||
def self.parse_wiki_titles(text)
|
||||
html = DTextRagel.parse(text)
|
||||
fragment = Nokogiri::HTML.fragment(html)
|
||||
@@ -142,6 +186,9 @@ class DText
|
||||
titles.uniq
|
||||
end
|
||||
|
||||
# Return a list of external links mentioned in a string of DText.
|
||||
# @param text [String] the string of DText
|
||||
# @return [Array<String>] the list of external URLs
|
||||
def self.parse_external_links(text)
|
||||
html = DTextRagel.parse(text)
|
||||
fragment = Nokogiri::HTML.fragment(html)
|
||||
@@ -150,6 +197,10 @@ class DText
|
||||
links.uniq
|
||||
end
|
||||
|
||||
# Return whether the two strings of DText have the same set of links.
|
||||
# @param a [String] a string of DText
|
||||
# @param b [String] a string of DText
|
||||
# @return [Boolean]
|
||||
def self.dtext_links_differ?(a, b)
|
||||
Set.new(parse_wiki_titles(a)) != Set.new(parse_wiki_titles(b)) ||
|
||||
Set.new(parse_external_links(a)) != Set.new(parse_external_links(b))
|
||||
@@ -159,6 +210,10 @@ class DText
|
||||
# the capitalization of the old tag when rewriting it to the new tag, but if
|
||||
# we can't determine how the new tag should be capitalized based on some
|
||||
# simple heuristics, then we skip rewriting the tag.
|
||||
# @param dtext [String] the DText input
|
||||
# @param old_name [String] the old wiki name
|
||||
# @param new_name [String] the new wiki name
|
||||
# @return [String] the DText output
|
||||
def self.rewrite_wiki_links(dtext, old_name, new_name)
|
||||
old_name = old_name.downcase.squeeze("_").tr("_", " ").strip
|
||||
new_name = new_name.downcase.squeeze("_").tr("_", " ").strip
|
||||
@@ -201,6 +256,10 @@ class DText
|
||||
end
|
||||
end
|
||||
|
||||
# Remove all [<tag>] blocks from the DText.
|
||||
# @param string [String] the DText input
|
||||
# @param tag [String] the type of block to remove
|
||||
# @return [String] the DText output
|
||||
def self.strip_blocks(string, tag)
|
||||
n = 0
|
||||
stripped = ""
|
||||
@@ -229,12 +288,18 @@ class DText
|
||||
stripped.strip
|
||||
end
|
||||
|
||||
# Remove all DText formatting from a string of DText, converting it to plain text.
|
||||
# @param dtext [String] the DText input
|
||||
# @return [String] the plain text output
|
||||
def self.strip_dtext(dtext)
|
||||
html = DTextRagel.parse(dtext)
|
||||
text = to_plaintext(html)
|
||||
text
|
||||
end
|
||||
|
||||
# Remove all formatting from a string of HTML, converting it to plain text.
|
||||
# @param html [String] the HTML input
|
||||
# @return [String] the plain text output
|
||||
def self.to_plaintext(html)
|
||||
text = from_html(html) do |node|
|
||||
case node.name
|
||||
@@ -250,10 +315,16 @@ class DText
|
||||
text.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
|
||||
end
|
||||
|
||||
# Convert DText formatting to Markdown.
|
||||
# @param dtext [String] the DText input
|
||||
# @return [String] the Markdown output
|
||||
def self.to_markdown(dtext)
|
||||
html_to_markdown(format_text(dtext))
|
||||
end
|
||||
|
||||
# Convert HTML to Markdown.
|
||||
# @param html [String] the HTML input
|
||||
# @return [String] the Markdown output
|
||||
def self.html_to_markdown(html)
|
||||
html = Nokogiri::HTML.fragment(html)
|
||||
|
||||
@@ -273,6 +344,10 @@ class DText
|
||||
end.join
|
||||
end
|
||||
|
||||
# Convert HTML to DText.
|
||||
# @param html [String] the HTML input
|
||||
# @param inline [Boolean] if true, convert <img> tags to plaintext
|
||||
# @return [String] the DText output
|
||||
def self.from_html(text, inline: false, &block)
|
||||
html = Nokogiri::HTML.fragment(text)
|
||||
|
||||
@@ -336,13 +411,19 @@ class DText
|
||||
dtext
|
||||
end
|
||||
|
||||
# extract the first paragraph `needle` occurs in.
|
||||
# Return the first paragraph the search string `needle` occurs in.
|
||||
# @param needle [String] the string to search for
|
||||
# @param dtext [String] the DText input
|
||||
# @return [String] the first paragraph mentioning the search string
|
||||
def self.extract_mention(dtext, needle)
|
||||
dtext = dtext.gsub(/\r\n|\r|\n/, "\n")
|
||||
excerpt = ActionController::Base.helpers.excerpt(dtext, needle, separator: "\n\n", radius: 1, omission: "")
|
||||
excerpt
|
||||
end
|
||||
|
||||
# Generate a short plain text excerpt from a DText string.
|
||||
# @param length [Integer] the max length of the output
|
||||
# @return [String] a plain text string
|
||||
def self.excerpt(text, length: 160)
|
||||
strip_dtext(text).split(/\r\n|\r|\n/).first.to_s.truncate(length)
|
||||
end
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
# The DanbooruLogger class handles logging messages to the Rails log and to NewRelic.
|
||||
#
|
||||
# @see https://guides.rubyonrails.org/debugging_rails_applications.html#the-logger
|
||||
# @see https://docs.newrelic.com
|
||||
class DanbooruLogger
|
||||
HEADERS = %w[referer sec-fetch-dest sec-fetch-mode sec-fetch-site sec-fetch-user]
|
||||
|
||||
# Log a message to the Rails log and to NewRelic.
|
||||
# @param message [String] the message to log
|
||||
# @param params [Hash] optional key-value data to log with the message
|
||||
def self.info(message, params = {})
|
||||
Rails.logger.info(message)
|
||||
|
||||
@@ -10,6 +17,13 @@ class DanbooruLogger
|
||||
end
|
||||
end
|
||||
|
||||
# Log an exception to the Rails log and to NewRelic. The `expected` flag is
|
||||
# used to separate expected exceptions, like search timeouts or auth failures,
|
||||
# from unexpected exceptions, like runtime errors, in the NewRelic error log.
|
||||
#
|
||||
# @param message [Exception] the exception to log
|
||||
# @param expected [Boolean] whether the exception was expected
|
||||
# @param params [Hash] optional key-value data to log with the exception
|
||||
def self.log(exception, expected: false, **params)
|
||||
if expected
|
||||
Rails.logger.info("#{exception.class}: #{exception.message}")
|
||||
@@ -23,6 +37,12 @@ class DanbooruLogger
|
||||
end
|
||||
end
|
||||
|
||||
# Log extra HTTP request data to NewRelic. Logs the user's IP, user agent,
|
||||
# request params, and session cookies.
|
||||
#
|
||||
# @param request the HTTP request
|
||||
# @param session the Rails session
|
||||
# @param user [User] the current user
|
||||
def self.add_session_attributes(request, session, user)
|
||||
add_attributes("request", { path: request.path })
|
||||
add_attributes("request.headers", header_params(request))
|
||||
@@ -31,6 +51,7 @@ class DanbooruLogger
|
||||
add_attributes("user", user_params(request, user))
|
||||
end
|
||||
|
||||
# Get logged HTTP headers from request.
|
||||
def self.header_params(request)
|
||||
headers = request.headers.to_h.select { |header, value| header.match?(/\AHTTP_/) }
|
||||
headers = headers.transform_keys { |header| header.delete_prefix("HTTP_").downcase }
|
||||
@@ -65,6 +86,7 @@ class DanbooruLogger
|
||||
|
||||
private_class_method
|
||||
|
||||
# @see https://docs.newrelic.com/docs/using-new-relic/data/customize-data/collect-custom-attributes/#ruby-att
|
||||
def self.add_custom_attributes(attributes)
|
||||
return unless defined?(::NewRelic)
|
||||
::NewRelic::Agent.add_custom_attributes(attributes)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# A DeviantArt API client.
|
||||
#
|
||||
# Authentication is via OAuth2 with the client credentials grant. Register a
|
||||
# new app at https://www.deviantart.com/developers/ to obtain a client_id and
|
||||
# client_secret. The app doesn't need to be published.
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# Builds an HTML diff between two pieces of text.
|
||||
class DiffBuilder
|
||||
attr_reader :this_text, :that_text, :pattern
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# A Discord API Client.
|
||||
#
|
||||
# @see https://discord.com/developers/docs/intro
|
||||
class DiscordApiClient
|
||||
extend Memoist
|
||||
|
||||
|
||||
@@ -1,3 +1,21 @@
|
||||
# The parent class for a Discord slash command. Each slash command handled by
|
||||
# Danbooru is a subclass of this class. Subclasses should set the {#name},
|
||||
# {#description}, and {#options} class attributes defining the slash command's
|
||||
# parameters, then implement the {#call} method to return a response to the
|
||||
# command.
|
||||
#
|
||||
# The lifecycle of a Discord slash command is this:
|
||||
#
|
||||
# * First we register the command with Discord with an API call ({DiscordApiClient#register_slash_command}).
|
||||
# * Then whenever someone uses the command, Discord sends us a HTTP request to
|
||||
# https://danbooru.donmai.us/webhooks/receive.
|
||||
# * We validate the request really came from Discord, then return the
|
||||
# command's response.
|
||||
#
|
||||
# @abstract
|
||||
# @see DiscordApiClient
|
||||
# @see WebhooksController#receive
|
||||
# @see https://discord.com/developers/docs/interactions/slash-commands
|
||||
class DiscordSlashCommand
|
||||
class WebhookVerificationError < StandardError; end
|
||||
|
||||
@@ -47,7 +65,7 @@ class DiscordSlashCommand
|
||||
end
|
||||
|
||||
concerning :HelperMethods do
|
||||
# The parameters passed to the command. A hash.
|
||||
# @return [Hash] The parameters passed to the slash command by the Discord user.
|
||||
def params
|
||||
@params ||= data.dig(:data, :options).to_a.map do |opt|
|
||||
[opt[:name], opt[:value]]
|
||||
|
||||
@@ -1,14 +1,25 @@
|
||||
require 'resolv'
|
||||
|
||||
# Validates that an email address is well-formed, is deliverable, and is not a
|
||||
# disposable or throwaway email address. Also normalizes equivalent addresses to
|
||||
# a single canonical form, so that users can't use different forms of the same
|
||||
# address to register multiple accounts.
|
||||
module EmailValidator
|
||||
module_function
|
||||
|
||||
# https://www.regular-expressions.info/email.html
|
||||
EMAIL_REGEX = /\A[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\z/
|
||||
|
||||
# Sites that ignore dots in email addresses, e.g. where `te.st@gmail.com` is
|
||||
# the same as `test@gmail.com`.
|
||||
IGNORE_DOTS = %w[gmail.com]
|
||||
|
||||
# Sites that allow plus addressing, e.g. `test+nospam@gmail.com`.
|
||||
# @see https://en.wikipedia.org/wiki/Email_address#Subaddressing
|
||||
IGNORE_PLUS_ADDRESSING = %w[gmail.com hotmail.com outlook.com live.com]
|
||||
IGNORE_MINUS_ADDRESSING = %w[yahoo.com]
|
||||
|
||||
# Sites that have multiple domains mapping to the same logical email address.
|
||||
CANONICAL_DOMAINS = {
|
||||
"googlemail.com" => "gmail.com",
|
||||
"hotmail.co.uk" => "outlook.com",
|
||||
@@ -66,10 +77,16 @@ module EmailValidator
|
||||
"gmx.us" => "gmx.net",
|
||||
}
|
||||
|
||||
# Returns true if it's okay to connect to port 25. Disabled outside of
|
||||
# production because many home ISPs blackhole port 25.
|
||||
def smtp_enabled?
|
||||
Rails.env.production?
|
||||
end
|
||||
|
||||
# Normalize an email address by stripping out plus addressing and dots, if
|
||||
# applicable, and rewriting the domain to a canonical domain.
|
||||
# @param address [String] the email address to normalize
|
||||
# @return [String] the normalized address
|
||||
def normalize(address)
|
||||
return nil unless address.count("@") == 1
|
||||
|
||||
@@ -83,10 +100,16 @@ module EmailValidator
|
||||
"#{name}@#{domain}"
|
||||
end
|
||||
|
||||
# Returns true if the email address is correctly formatted.
|
||||
# @param [String] the email address
|
||||
# @return [Boolean]
|
||||
def is_valid?(address)
|
||||
address.match?(EMAIL_REGEX)
|
||||
end
|
||||
|
||||
# Returns true if the email is a throwaway or disposable email address.
|
||||
# @param [String] the email address
|
||||
# @return [Boolean]
|
||||
def is_restricted?(address)
|
||||
return false if Danbooru.config.email_domain_verification_list.blank?
|
||||
|
||||
@@ -96,6 +119,11 @@ module EmailValidator
|
||||
true
|
||||
end
|
||||
|
||||
# Returns true if the email can't be delivered. Checks if the domain has an MX
|
||||
# record and responds to the RCPT TO command.
|
||||
# @param to_address [String] the email address to check
|
||||
# @param from_address [String] the email address to check from
|
||||
# @return [Boolean]
|
||||
def undeliverable?(to_address, from_address: Danbooru.config.contact_email, timeout: 3)
|
||||
mail_server = mx_domain(to_address, timeout: timeout)
|
||||
mail_server.nil? || rcpt_to_failed?(to_address, from_address, mail_server, timeout: timeout)
|
||||
@@ -103,6 +131,13 @@ module EmailValidator
|
||||
false
|
||||
end
|
||||
|
||||
# Returns true if the email can't be delivered. Sends a RCPT TO command over
|
||||
# port 25 to check if the mailbox exists.
|
||||
# @param to_address [String] the email address to check
|
||||
# @param from_address [String] the email address to check from
|
||||
# @param mail_server [String] the DNS name of the SMTP server
|
||||
# @param timeout [Integer] the network timeout
|
||||
# @return [Boolean]
|
||||
def rcpt_to_failed?(to_address, from_address, mail_server, timeout: nil)
|
||||
return false unless smtp_enabled?
|
||||
|
||||
@@ -121,6 +156,11 @@ module EmailValidator
|
||||
end
|
||||
end
|
||||
|
||||
# Does a DNS MX record lookup of the domain in the email address and returns the
|
||||
# name of the mail server, if it exists.
|
||||
# @param to_address [String] the email address to check
|
||||
# @param timeout [Integer] the network timeout
|
||||
# @return [String] the DNS name of the mail server
|
||||
def mx_domain(to_address, timeout: nil)
|
||||
domain = Mail::Address.new(to_address).domain
|
||||
|
||||
|
||||
@@ -1,21 +1,48 @@
|
||||
# Parse the output of the Postgres EXPLAIN command to get how many rows Postgres
|
||||
# *thinks* a SQL query will return. This is an estimate, and only accurate for
|
||||
# queries that have a single condition. If the query has multiple conditions,
|
||||
# then Postgres will assume they're independent (have no overlap), which is not
|
||||
# always the case.
|
||||
#
|
||||
# Used by {PostQueryBuilder#fast_count} to get a fast post count estimate for
|
||||
# certain single metatag searches.
|
||||
#
|
||||
# ExplainParser.new(Post.all).query_plan
|
||||
# => EXPLAIN (FORMAT JSON) SELECT "posts".* FROM "posts"
|
||||
# => {
|
||||
# "Node Type"=>"Seq Scan",
|
||||
# "Parallel Aware"=>false,
|
||||
# "Relation Name"=>"posts",
|
||||
# "Alias"=>"posts",
|
||||
# "Startup Cost"=>0.0,
|
||||
# "Total Cost"=>780900.02,
|
||||
# "Plan Rows"=>4413102,
|
||||
# "Plan Width"=>1268
|
||||
# }
|
||||
#
|
||||
# @see https://www.postgresql.org/docs/current/sql-explain.html
|
||||
class ExplainParser
|
||||
extend Memoist
|
||||
attr_reader :relation
|
||||
|
||||
# @param the ActiveRecord relation
|
||||
def initialize(relation)
|
||||
@relation = relation
|
||||
end
|
||||
|
||||
# @return [Hash] the Postgres query plan
|
||||
def query_plan
|
||||
result = ApplicationRecord.connection.select_one("EXPLAIN (FORMAT JSON) #{sql}")
|
||||
json = JSON.parse(result["QUERY PLAN"])
|
||||
json.first["Plan"]
|
||||
end
|
||||
|
||||
# @return [Integer] the number of rows Postgres *thinks* the query will return
|
||||
def row_count
|
||||
query_plan["Plan Rows"]
|
||||
end
|
||||
|
||||
# @return [String] the query's SQL
|
||||
def sql
|
||||
relation.reorder(nil).to_sql
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# Add a post to a forum topic.
|
||||
class ForumUpdater
|
||||
attr_reader :forum_topic
|
||||
|
||||
|
||||
@@ -1,13 +1,20 @@
|
||||
# See also config/initializers/types.rb
|
||||
|
||||
# Ensure ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Inet is loaded in
|
||||
# the dev environment when eager loading is disabled.
|
||||
require "active_record/connection_adapters/postgresql_adapter"
|
||||
|
||||
# Define a custom IP address type for IP columns in the database. IP columns
|
||||
# will be serialized and deserialized as a {Danbooru::IpAddress} object.
|
||||
#
|
||||
# @see config/initializers/types.rb
|
||||
# @see https://www.bigbinary.com/blog/rails-5-attributes-api
|
||||
class IpAddressType < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Inet
|
||||
# @param value [String] the IP address from the database
|
||||
def cast(value)
|
||||
return nil if value.blank?
|
||||
super(Danbooru::IpAddress.new(value))
|
||||
end
|
||||
|
||||
# @param [Danbooru::IpAddress] the IP address object
|
||||
def serialize(value)
|
||||
value&.to_string
|
||||
end
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# API client for https://ipregistry.co/
|
||||
|
||||
# An API client for https://ipregistry.co. Looks up IP address information,
|
||||
# including the geolocation and whether the IP is a proxy.
|
||||
#
|
||||
# @see https://ipregistry.co/docs
|
||||
class IpLookup
|
||||
extend Memoist
|
||||
|
||||
|
||||
@@ -1,13 +1,21 @@
|
||||
# An API client for Danbooru's internal IQDB instance. Can add images, remove
|
||||
# images, and search for images in IQDB.
|
||||
#
|
||||
# @see https://github.com/danbooru/iqdb
|
||||
class IqdbClient
|
||||
class Error < StandardError; end
|
||||
attr_reader :iqdb_url, :http
|
||||
|
||||
# Create a new IQDB API client.
|
||||
# @param iqdb_url [String] the base URL of the IQDB server
|
||||
# @param http [Danbooru::Http] the HTTP client to use
|
||||
def initialize(iqdb_url: Danbooru.config.iqdb_url.to_s, http: Danbooru::Http.new)
|
||||
@iqdb_url = iqdb_url.chomp("/")
|
||||
@http = http
|
||||
end
|
||||
|
||||
concerning :QueryMethods do
|
||||
# Search for an image by file, URL, or post ID.
|
||||
def search(post_id: nil, file: nil, url: nil, image_url: nil, file_url: nil, similarity: 0.0, high_similarity: 65.0, limit: 20)
|
||||
limit = limit.to_i.clamp(1, 1000)
|
||||
similarity = similarity.to_f.clamp(0.0, 100.0)
|
||||
@@ -37,6 +45,10 @@ class IqdbClient
|
||||
file.try(:close)
|
||||
end
|
||||
|
||||
# Download an URL to a file.
|
||||
# @param url [String] the URL to download
|
||||
# @param type [Symbol] the type of URL to download (:preview_url or full :image_url)
|
||||
# @return [MediaFile] the downloaded file
|
||||
def download(url, type)
|
||||
strategy = Sources::Strategies.find(url)
|
||||
download_url = strategy.send(type)
|
||||
@@ -44,8 +56,12 @@ class IqdbClient
|
||||
file
|
||||
end
|
||||
|
||||
def decorate_posts(json)
|
||||
post_ids = json.map { |match| match["post_id"] }
|
||||
# Transform the JSON returned by IQDB to add the full post data for each
|
||||
# match.
|
||||
# @param matches [Array<Hash>] the array of IQDB matches
|
||||
# @return [Array<Hash>] the array of IQDB matches, with post data
|
||||
def decorate_posts(matches)
|
||||
post_ids = matches.map { |match| match["post_id"] }
|
||||
posts = Post.where(id: post_ids).group_by(&:id).transform_values(&:first)
|
||||
|
||||
json.map do |match|
|
||||
@@ -55,6 +71,8 @@ class IqdbClient
|
||||
end
|
||||
end
|
||||
|
||||
# Add a post to IQDB.
|
||||
# @param post [Post] the post to add
|
||||
def add_post(post)
|
||||
return unless post.has_preview?
|
||||
preview_file = post.file(:preview)
|
||||
@@ -62,20 +80,31 @@ class IqdbClient
|
||||
end
|
||||
|
||||
concerning :HttpMethods do
|
||||
# Search for an image in IQDB.
|
||||
# @param file [File] the image to search
|
||||
def query(file, limit: 20)
|
||||
file = HTTP::FormData::File.new(file)
|
||||
request(:post, "query", form: { file: file }, params: { limit: limit })
|
||||
end
|
||||
|
||||
# Add a post to IQDB.
|
||||
# @param post_id [Integer] the post to add
|
||||
# @param file [File] the image to add
|
||||
def add(post_id, file)
|
||||
file = HTTP::FormData::File.new(file)
|
||||
request(:post, "images/#{post_id}", form: { file: file })
|
||||
end
|
||||
|
||||
# Remove an image from IQDB.
|
||||
# @param post_id [Integer] the post to remove
|
||||
def remove(post_id)
|
||||
request(:delete, "images/#{post_id}")
|
||||
end
|
||||
|
||||
# Send a request to IQDB.
|
||||
# @param method [String] the HTTP method
|
||||
# @param url [String] the IQDB url
|
||||
# @param options [Hash] the URL params to send
|
||||
def request(method, url, **options)
|
||||
return [] if iqdb_url.blank? # do nothing if iqdb isn't configured
|
||||
response = http.timeout(30).send(method, "#{iqdb_url}/#{url}", **options)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# An API client for Mastodon.
|
||||
#
|
||||
# @see https://docs.joinmastodon.org/api
|
||||
class MastodonApiClient
|
||||
extend Memoist
|
||||
attr_reader :json
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# An API client for the NicoSeiga XML API.
|
||||
class NicoSeigaApiClient
|
||||
extend Memoist
|
||||
XML_API = "https://seiga.nicovideo.jp/api"
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# Sanitizes the HTML used in notes. Only safe HTML tags, HTML attributes, and
|
||||
# CSS properties are allowed.
|
||||
module NoteSanitizer
|
||||
ALLOWED_ELEMENTS = %w[
|
||||
code center tn h1 h2 h3 h4 h5 h6 a span div blockquote br p ul li ol em
|
||||
@@ -50,6 +52,9 @@ module NoteSanitizer
|
||||
vertical-align
|
||||
]
|
||||
|
||||
# Sanitize a string of HTML.
|
||||
# @param text [String] the HTML to sanitize
|
||||
# @return [String] the sanitized HTML
|
||||
def self.sanitize(text)
|
||||
text.gsub!(/<( |-|3|:|>|\Z)/, "<\\1")
|
||||
|
||||
@@ -76,6 +81,8 @@ module NoteSanitizer
|
||||
)
|
||||
end
|
||||
|
||||
# Convert absolute Danbooru links inside notes to relative links.
|
||||
# https://danbooru.donmai.us/posts/1 is converted to /posts/1.
|
||||
def self.relativize_links(node:, **env)
|
||||
return unless node.name == "a" && node["href"].present?
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
# Notes on Pixiv's AJAX API:
|
||||
# A client for Pixiv's AJAX API.
|
||||
#
|
||||
# Notes on the API:
|
||||
#
|
||||
# * The user agent must be spoofed as a browser user agent, otherwise the API
|
||||
# will return an error.
|
||||
@@ -359,23 +361,40 @@ class PixivAjaxClient
|
||||
|
||||
attr_reader :phpsessid, :http
|
||||
|
||||
# @param phpsessid [String] the Pixiv login cookie
|
||||
# @param http [Danbooru::Http] the HTTP client to use for Pixiv
|
||||
def initialize(phpsessid, http: Danbooru::Http.new)
|
||||
@phpsessid = phpsessid
|
||||
@http = http
|
||||
end
|
||||
|
||||
# Return the illust data for a Pixiv illustration.
|
||||
# @see https://www.pixiv.net/ajax/illust/87598468
|
||||
# @param illust_id [Integer] the Pixiv illustration id
|
||||
# @return [Hash] the illustration data
|
||||
def illust(illust_id)
|
||||
get("https://www.pixiv.net/ajax/illust/#{illust_id}").with_indifferent_access
|
||||
end
|
||||
|
||||
# Return the data for a Pixiv manga illust with multiple pages.
|
||||
# @see https://www.pixiv.net/ajax/illust/87598468/pages
|
||||
# @param illust_id [Integer] the Pixiv illustration id
|
||||
# @return [Hash] the illustration data
|
||||
def pages(illust_id)
|
||||
get("https://www.pixiv.net/ajax/illust/#{illust_id}/pages")
|
||||
end
|
||||
|
||||
# Return the data for a Pixiv ugoira.
|
||||
# @see https://www.pixiv.net/ajax/illust/74932152/ugoira_meta
|
||||
# @param illust_id [Integer] the Pixiv illustration id
|
||||
# @return [Hash] the ugoira data
|
||||
def ugoira_meta(illust_id)
|
||||
get("https://www.pixiv.net/ajax/illust/#{illust_id}/ugoira_meta").with_indifferent_access
|
||||
end
|
||||
|
||||
# Perform a GET request for a Pixiv URL.
|
||||
# @param url [String] the Pixiv URL
|
||||
# @return [Hash] the parsed response, or blank on error
|
||||
def get(url)
|
||||
response = client.cache(1.minute).get(url)
|
||||
|
||||
@@ -387,6 +406,7 @@ class PixivAjaxClient
|
||||
end
|
||||
end
|
||||
|
||||
# @return [Danbooru::Http] the HTTP client used for Pixiv
|
||||
def client
|
||||
@client ||= http.headers("User-Agent": USER_AGENT).cookies(PHPSESSID: phpsessid)
|
||||
end
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# Posts new appeals to the Deletion appeal thread once per hour.
|
||||
#
|
||||
# @see DanbooruMaintenance#hourly
|
||||
module PostAppealForumUpdater
|
||||
APPEAL_TOPIC_TITLE = "Deletion appeal thread"
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# Delete posts that were unapproved after three days.
|
||||
module PostPruner
|
||||
module_function
|
||||
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
# Define a custom validator for tag names. Tags must be plain ASCII, no spaces,
|
||||
# no redundant underscores, no conflicts with metatags, and can't begin with
|
||||
# certain special characters.
|
||||
#
|
||||
# @example
|
||||
# validates :name, tag_name: true
|
||||
#
|
||||
# @see https://guides.rubyonrails.org/active_record_validations.html#custom-validators
|
||||
class TagNameValidator < ActiveModel::EachValidator
|
||||
MAX_TAG_LENGTH = 170
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
# Removes tag aliases and implications if they haven't had any new uploads in
|
||||
# the last two years. Runs weekly. Posts a message to the forum when aliases or
|
||||
# implications are retired.
|
||||
#
|
||||
# @see DanbooruMaintenance#weekly
|
||||
module TagRelationshipRetirementService
|
||||
module_function
|
||||
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
# Define a custom validator for user names.
|
||||
#
|
||||
# @example
|
||||
# validates :name, user_name: true
|
||||
#
|
||||
# @see https://guides.rubyonrails.org/active_record_validations.html#custom-validators
|
||||
class UserNameValidator < ActiveModel::EachValidator
|
||||
def validate_each(rec, attr, value)
|
||||
name = value
|
||||
|
||||
Reference in New Issue
Block a user