jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into attribute-searching

Browse Source
This commit is contained in:
evazion
2020-08-17 14:23:00 -05:00
committed by GitHub
parent 7291de42f4 d9f0a1c5e3
commit 5db11a0b5f
155 changed files with 2834 additions and 2169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
app/logical/concerns/searchable.rb
View File

@@ -163,7 +163,10 @@ module Searchable
type = column.type || reflect_on_association(name)&.class_name
if column.try(:array?)
return search_array_attribute(name, type, params)
subtype = type
type = :array
elsif defined_enums.has_key?(name.to_s)
type = :enum
end
case type
@@ -181,6 +184,10 @@ module Searchable
numeric_attribute_matches(name, params[name])
when :inet
search_inet_attribute(name, params)
when :enum
search_enum_attribute(name, params)
when :array
search_array_attribute(name, subtype, params)
else
raise NotImplementedError, "unhandled attribute type: #{name}" if type.blank?
search_includes(name, params, type)
@@ -279,6 +286,19 @@ module Searchable
relation
end
def search_enum_attribute(name, params)
relation = all
if params[name].present?
value = params[name].split(/[, ]+/).map(&:downcase)
relation = relation.where(name => value)
elsif params["#{name}_id"].present?
relation = relation.numeric_attribute_matches(name, params["#{name}_id"])
end
relation
end
def search_array_attribute(name, type, params)
relation = all

1
app/logical/danbooru/http.rb
View File

@@ -1,6 +1,7 @@
require "danbooru/http/html_adapter"
require "danbooru/http/xml_adapter"
require "danbooru/http/cache"
require "danbooru/http/logger"
require "danbooru/http/redirector"
require "danbooru/http/retriable"
require "danbooru/http/session"

35
app/logical/danbooru/http/logger.rb Normal file
View File

@@ -0,0 +1,35 @@
module Danbooru
class Http
class Logger < HTTP::Feature
HTTP::Options.register_feature :logger, self
attr_reader :logger
def initialize(logger: ::Logger.new(STDOUT))
@logger = logger
end
def perform(request, &block)
log_request(request)
response = yield request
log_response(request, response)
response
end
def log_request(request)
logger.info do
verb = request.verb.to_s.upcase
headers = request.headers.map { |name, value| "#{name}: #{value}" }.join("\n")
"> #{verb} #{request.uri}\n#{headers}\n"
end
end
def log_response(request, response)
logger.info do
headers = response.headers.map { |name, value| "#{name}: #{value}" }.join("\n")
"< #{response.status.to_i} | #{request.uri}\n#{headers}\n"
end
end
end
end
end

11
app/logical/danbooru_maintenance.rb
View File

@@ -3,13 +3,14 @@ module DanbooruMaintenance
def hourly
safely { Upload.prune! }
safely { PostPruner.prune! }
safely { PostAppealForumUpdater.update_forum! }
safely { regenerate_post_counts! }
end
def daily
safely { PostPruner.new.prune! }
safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all }
safely { PostDisapproval.prune! }
safely { regenerate_post_counts! }
safely { TokenBucket.prune! }
safely { BulkUpdateRequestPruner.warn_old }
safely { BulkUpdateRequestPruner.reject_expired }
@@ -35,8 +36,12 @@ module DanbooruMaintenance
def safely(&block)
ActiveRecord::Base.connection.execute("set statement_timeout = 0")
yield
CurrentUser.scoped(User.system, "127.0.0.1") do
yield
end
rescue StandardError => exception
DanbooruLogger.log(exception)
raise exception if Rails.env.test?
end
end

26
app/logical/post_appeal_forum_updater.rb Normal file
View File

@@ -0,0 +1,26 @@
module PostAppealForumUpdater
APPEAL_TOPIC_TITLE = "Deletion appeal thread"
def self.update_forum!
return if pending_appeals.empty?
CurrentUser.scoped(User.system) do
topic = ForumTopic.order(:id).create_with(creator: User.system).find_or_create_by!(title: APPEAL_TOPIC_TITLE)
ForumPost.create!(creator: User.system, topic: topic, body: forum_post_body)
end
end
def self.pending_appeals
PostAppeal.pending.where(created_at: (1.hour.ago..Time.zone.now)).order(post_id: :asc)
end
def self.forum_post_body
pending_appeals.map do |appeal|
if appeal.reason.present?
"post ##{appeal.post_id}: #{appeal.reason}"
else
"post ##{appeal.post_id}"
end
end.join("\n")
end
end

34
app/logical/post_pruner.rb
View File

@@ -1,37 +1,27 @@
class PostPruner
module PostPruner
module_function
def prune!
prune_pending!
prune_flagged!
prune_mod_actions!
prune_appealed!
end
protected
def prune_pending!
CurrentUser.scoped(User.system, "127.0.0.1") do
Post.where("is_deleted = ? and is_pending = ? and created_at < ?", false, true, 3.days.ago).each do |post|
post.delete!("Unapproved in three days")
rescue PostFlag::Error
# swallow
end
Post.pending.expired.each do |post|
post.delete!("Unapproved in three days", user: User.system)
end
end
def prune_flagged!
CurrentUser.scoped(User.system, "127.0.0.1") do
Post.where("is_deleted = ? and is_flagged = ?", false, true).each do |post|
if post.flags.unresolved.old.any?
begin
post.delete!("Unapproved in three days after returning to moderation queue")
rescue PostFlag::Error
# swallow
end
end
end
PostFlag.expired.each do |flag|
flag.post.delete!("Unapproved in three days after returning to moderation queue", user: User.system)
end
end
def prune_mod_actions!
ModAction.where(["creator_id = ? and description like ?", User.system.id, "deleted post %"]).destroy_all
def prune_appealed!
PostAppeal.expired.each do |appeal|
appeal.post.delete!("Unapproved in three days after returning to moderation queue", user: User.system)
end
end
end

79
app/logical/post_query_builder.rb
View File

@@ -6,7 +6,7 @@ class PostQueryBuilder
COUNT_METATAGS = %w[
comment_count deleted_comment_count active_comment_count
note_count deleted_note_count active_note_count
flag_count resolved_flag_count unresolved_flag_count
flag_count
child_count deleted_child_count active_child_count
pool_count deleted_pool_count active_pool_count series_pool_count collection_pool_count
appeal_count approval_count replacement_count
@@ -274,8 +274,10 @@ class PostQueryBuilder
Post.pending
when "flagged"
Post.flagged
when "appealed"
Post.appealed
when "modqueue"
Post.pending_or_flagged
Post.in_modqueue
when "deleted"
Post.deleted
when "banned"
@@ -283,7 +285,7 @@ class PostQueryBuilder
when "active"
Post.active
when "unmoderated"
Post.pending_or_flagged.available_for_moderation(current_user, hidden: false)
Post.in_modqueue.available_for_moderation(current_user, hidden: false)
when "all", "any"
Post.all
else
@@ -307,7 +309,7 @@ class PostQueryBuilder
Post.where(parent: nil)
when "any"
Post.where.not(parent: nil)
when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
when "pending", "flagged", "appealed", "modqueue", "deleted", "banned", "active", "unmoderated"
Post.where.not(parent: nil).where(parent: status_matches(parent))
when /\A\d+\z/
Post.where(id: parent).or(Post.where(parent: parent))
@@ -322,7 +324,7 @@ class PostQueryBuilder
Post.where(has_children: false)
when "any"
Post.where(has_children: true)
when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
when "pending", "flagged", "appealed", "modqueue", "deleted", "banned", "active", "unmoderated"
Post.where(has_children: true).where(children: status_matches(child))
else
Post.none
@@ -330,8 +332,9 @@ class PostQueryBuilder
end
def source_matches(source, quoted = false)
case source.downcase
in "none" unless quoted
if source.empty?
Post.where_like(:source, "")
elsif source.downcase == "none" && !quoted
Post.where_like(:source, "")
else
Post.where_ilike(:source, source + "*")
@@ -606,10 +609,10 @@ class PostQueryBuilder
.order("contributor_fav_count DESC, posts.fav_count DESC, posts.id DESC")
when "modqueue", "modqueue_desc"
relation = relation.left_outer_joins(:flags).order(Arel.sql("GREATEST(posts.created_at, post_flags.created_at) DESC, posts.id DESC"))
relation = relation.with_queued_at.order("queued_at DESC, posts.id DESC")
when "modqueue_asc"
relation = relation.left_outer_joins(:flags).order(Arel.sql("GREATEST(posts.created_at, post_flags.created_at) ASC, posts.id ASC"))
relation = relation.with_queued_at.order("queued_at ASC, posts.id ASC")
when "none"
relation = relation.reorder(nil)
@@ -642,14 +645,7 @@ class PostQueryBuilder
if scanner.scan(/(-)?(#{METATAGS.join("|")}):/io)
operator = scanner.captures.first
metatag = scanner.captures.second.downcase
if scanner.scan(/"(.+)"/) || scanner.scan(/'(.+)'/)
value = scanner.captures.first
quoted = true
else
value = scanner.scan(/[^ ]*/)
quoted = false
end
value, quoted = scan_string(scanner)
if metatag.in?(COUNT_METATAG_SYNONYMS)
metatag = metatag.singularize + "_count"
@@ -673,23 +669,41 @@ class PostQueryBuilder
terms
end
def scan_string(scanner)
if scanner.scan(/"((?:\\"|[^"])*)"/)
value = scanner.captures.first.gsub(/\\(.)/) { $1 }
quoted = true
elsif scanner.scan(/'((?:\\'|[^'])*)'/)
value = scanner.captures.first.gsub(/\\(.)/) { $1 }
quoted = true
else
value = scanner.scan(/(\\ |[^ ])*/)
value = value.gsub(/\\ /) { " " }
quoted = false
end
[value, quoted]
end
def split_query
terms.map do |term|
if term.type == :metatag && !term.negated && !term.quoted
"#{term.name}:#{term.value}"
elsif term.type == :metatag && !term.negated && term.quoted
"#{term.name}:\"#{term.value}\""
elsif term.type == :metatag && term.negated && !term.quoted
"-#{term.name}:#{term.value}"
elsif term.type == :metatag && term.negated && term.quoted
"-#{term.name}:\"#{term.value}\""
elsif term.type == :tag && term.negated
"-#{term.name}"
elsif term.type == :tag && term.optional
"~#{term.name}"
elsif term.type == :tag
term.name
type, name, value = term.type, term.name, term.value
str = ""
str += "-" if term.negated
str += "~" if term.optional
if type == :tag
str += name
elsif type == :metatag && (term.quoted || value.include?(" "))
value = value.gsub(/\\/) { '\\\\' }
value = value.gsub(/"/) { '\\"' }
str += "#{name}:\"#{value}\""
elsif type == :metatag
str += "#{name}:#{value}"
end
str
end
end
@@ -898,8 +912,9 @@ class PostQueryBuilder
metatags
end
# XXX unify with PostSets::Post#show_deleted?
def hide_deleted?
has_status_metatag = select_metatags(:status).any? { |metatag| metatag.value.downcase.in?(%w[deleted active any all]) }
has_status_metatag = select_metatags(:status).any? { |metatag| metatag.value.downcase.in?(%w[deleted active any all unmoderated modqueue appealed]) }
hide_deleted_posts? && !has_status_metatag
end
end

22
app/logical/post_sets/post.rb
View File

@@ -59,6 +59,12 @@ module PostSets
posts.any? {|x| x.rating == "e"}
end
def shown_posts
shown_posts = posts.select(&:visible?)
shown_posts = shown_posts.reject(&:is_deleted?) unless show_deleted?
shown_posts
end
def hidden_posts
posts.reject(&:visible?)
end
@@ -136,24 +142,22 @@ module PostSets
def post_previews_html(template)
html = ""
if none_shown
if shown_posts.empty?
return template.render("post_sets/blank")
end
posts.each do |post|
html << PostPresenter.preview(post, show_cropped: true, tags: tag_string)
shown_posts.each do |post|
html << PostPresenter.preview(post, show_deleted: show_deleted?, show_cropped: true, tags: tag_string)
html << "\n"
end
html.html_safe
end
def not_shown(post)
post.is_deleted? && tag_string !~ /status:(?:all|any|deleted|banned)/
end
def none_shown
posts.reject {|post| not_shown(post) }.empty?
def show_deleted?
query.select_metatags("status").any? do |metatag|
metatag.value.in?(%w[all any active unmoderated modqueue deleted appealed])
end
end
concerning :TagListMethods do

4
app/logical/sources/strategies/deviant_art.rb
View File

@@ -77,6 +77,10 @@ module Sources
FAVME = %r{\Ahttps?://(?:www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
def self.enabled?
Danbooru.config.deviantart_client_id.present? && Danbooru.config.deviantart_client_secret.present?
end
def domains
["deviantart.net", "deviantart.com", "fav.me"]
end

4
app/logical/sources/strategies/nico_seiga.rb
View File

@@ -50,6 +50,10 @@ module Sources
PROFILE_PAGE = %r{\Ahttps?://seiga\.nicovideo\.jp/user/illust/(?<artist_id>\d+)}i
def self.enabled?
Danbooru.config.nico_seiga_login.present? && Danbooru.config.nico_seiga_password.present?
end
def domains
["nicoseiga.jp", "nicovideo.jp"]
end

40
app/logical/sources/strategies/nijie.rb
View File

@@ -64,6 +64,10 @@ module Sources
DIR = %r{(?:\d+/)?(?:__rs_\w+/)?nijie_picture(?:/diff/main)?}
IMAGE_URL = %r{#{IMAGE_BASE_URL}/#{DIR}/#{Regexp.union(FILENAME1, FILENAME2, FILENAME3)}\.\w+\z}i
def self.enabled?
Danbooru.config.nijie_login.present? && Danbooru.config.nijie_password.present?
end
def domains
["nijie.info", "nijie.net"]
end
@@ -176,23 +180,37 @@ module Sources
end
def page
return nil if page_url.blank?
return nil if page_url.blank? || client.blank?
http = Danbooru::Http.new
form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password }
# XXX `retriable` must come after `cache` so that retries don't return cached error responses.
response = http.cache(1.hour).use(retriable: { max_retries: 20 }).post("https://nijie.info/login_int.php", form: form)
DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200
return nil unless response.status == 200
response = http.cookies(R18: 1).cache(1.minute).get(page_url)
response = client.cache(1.minute).get(page_url)
return nil unless response.status == 200
response&.parse
end
memoize :page
def client
http = Danbooru::Http.new.timeout(60)
cookie = Cache.get("nijie-session-cookie", 1.week) do
login_page = http.use(retriable: { max_retries: 20 }).get("https://nijie.info/login.php").parse
form = {
email: Danbooru.config.nijie_login,
password: Danbooru.config.nijie_password,
url: login_page.at("input[name='url']")["value"],
save: "on",
ticket: ""
}
response = http.use(retriable: { max_retries: 20 }).post("https://nijie.info/login_int.php", form: form)
DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200
return nil unless response.status == 200
response.cookies.select { |c| c.name == "NIJIEIJIEID" }.compact.first
end
http.cookies(NIJIEIJIEID: cookie, R18: 1)
end
memoize :client
end
end
end

4
app/logical/sources/strategies/pawoo.rb
View File

@@ -24,6 +24,10 @@ module Sources::Strategies
STATUS1 = %r{\A#{HOST}/web/statuses/(?<status_id>\d+)}
STATUS2 = %r{\A#{NAMED_PROFILE}/(?<status_id>\d+)}
def self.enabled?
Danbooru.config.pawoo_client_id.present? && Danbooru.config.pawoo_client_secret.present?
end
def domains
["pawoo.net"]
end

4
app/logical/sources/strategies/pixiv.rb
View File

@@ -65,6 +65,10 @@ module Sources
STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i
NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))}
def self.enabled?
Danbooru.config.pixiv_login.present? && Danbooru.config.pixiv_password.present?
end
def self.to_dtext(text)
if text.nil?
return nil

27
app/logical/sources/strategies/tumblr.rb
View File

@@ -7,6 +7,8 @@
# https://66.media.tumblr.com/5a2c3fe25c977e2281392752ab971c90/3dbfaec9b9e0c2e3-92/s500x750/4f92bbaaf95c0b4e7970e62b1d2e1415859dd659.png
#
# https://superboin.tumblr.com/post/141169066579/photoset_iframe/superboin/tumblr_o45miiAOts1u6rxu8/500/false
#
# https://make-do5.tumblr.com/post/619663949657423872 (extremely high res, extractable)
module Sources::Strategies
class Tumblr < Base
@@ -26,6 +28,11 @@ module Sources::Strategies
VIDEO = %r{\Ahttps?://(?:vtt|ve|va\.media)\.tumblr\.com/}i
POST = %r{\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)}i
NEW_HEADERS = {
"user-agent": Danbooru.config.canonical_app_name,
"accept": "text/html"
}
def self.enabled?
Danbooru.config.tumblr_consumer_key.present?
end
@@ -161,14 +168,22 @@ module Sources::Strategies
# http://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg
# => https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg
def find_largest(url, sizes: SIZES)
return url unless url =~ OLD_IMAGE
if url =~ OLD_IMAGE
candidates = sizes.map do |size|
"https://media.tumblr.com/#{$~[:dir]}#{$~[:filename]}_#{size}.#{$~[:ext]}"
end
candidates = sizes.map do |size|
"https://media.tumblr.com/#{$~[:dir]}#{$~[:filename]}_#{size}.#{$~[:ext]}"
end
candidates.find do |candidate|
http_exists?(candidate)
end
elsif url =~ %r{/s\d+x\d+/(\w+\.\w+)$}i
max_size = Integer.sqrt(Danbooru.config.max_image_resolution)
url = url.gsub(%r{/s\d+x\d+/\w+\.\w+$}i, "/s#{max_size}x#{max_size}/#{$1}")
candidates.find do |candidate|
http_exists?(candidate)
resp = Danbooru::Http.cache(1.minute).get(url, headers: NEW_HEADERS).parse
resp.at("img[src*='/s#{max_size}x#{max_size}/']")["src"]
else
url
end
end

5
app/logical/tag_category.rb
View File

@@ -25,11 +25,6 @@ class TagCategory
@@short_name_mapping ||= Hash[Danbooru.config.full_tag_config_info.map { |k, v| [v["short"], k] }]
end
# Returns a hash mapping for split_tag_list_html (presenters/tag_set_presenter.rb)
def header_mapping
@@header_mapping ||= Hash[Danbooru.config.full_tag_config_info.map { |k, v| [k, v["header"]] }]
end
# Returns a hash mapping for related tag buttons (javascripts/related_tag.js.erb)
def related_button_mapping
@@related_button_mapping ||= Hash[Danbooru.config.full_tag_config_info.map { |k, v| [k, v["relatedbutton"]] }]

21
app/logical/upload_limit.rb
View File

@@ -3,6 +3,8 @@ class UploadLimit
INITIAL_POINTS = 1000
MAXIMUM_POINTS = 10_000
APPEAL_COST = 3
DELETION_COST = 5
attr_reader :user
@@ -30,11 +32,20 @@ class UploadLimit
end
end
def used_upload_slots
pending = user.posts.pending
early_deleted = user.posts.deleted.where("created_at >= ?", 3.days.ago)
def maxed?
user.upload_points >= MAXIMUM_POINTS
end
pending.or(early_deleted).count
def used_upload_slots
pending_count = user.posts.pending.count
appealed_count = user.post_appeals.pending.count
early_deleted_count = user.posts.deleted.where("created_at >= ?", Danbooru.config.moderation_period.ago).count
pending_count + (early_deleted_count * DELETION_COST) + (appealed_count * APPEAL_COST)
end
def free_upload_slots
upload_slots - used_upload_slots
end
def upload_slots
@@ -111,6 +122,4 @@ class UploadLimit
points_for_next_level(n - 1)
end.sum
end
memoize :used_upload_slots
end

2
app/logical/upload_service/preprocessor.rb
View File

@@ -64,6 +64,8 @@ class UploadService
end
def start!
raise NotImplementedError, "No login credentials configured for #{strategy.site_name}." unless strategy.class.enabled?
if Utils.is_downloadable?(source)
if Post.system_tag_match("source:#{canonical_source}").where.not(id: original_post_id).exists?
raise ActiveRecord::RecordNotUnique, "A post with source #{canonical_source} already exists"

2
app/logical/upload_service/utils.rb
View File

@@ -72,6 +72,8 @@ class UploadService
raise "No file or source URL provided" if upload.source_url.blank?
strategy = Sources::Strategies.find(upload.source_url, upload.referer_url)
raise NotImplementedError, "No login credentials configured for #{strategy.site_name}." unless strategy.class.enabled?
file = strategy.download_file!
if strategy.data[:ugoira_frame_data].present?