jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into skeb

Browse Source
This commit is contained in:
evazion
2021-03-08 03:43:15 -06:00
committed by GitHub
parent dc97f4483b 66af4986c6
commit 9dd903d212
91 changed files with 840 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/logical/danbooru_maintenance.rb
View File

@@ -5,13 +5,13 @@ module DanbooruMaintenance
safely { Upload.prune! }
safely { PostPruner.prune! }
safely { PostAppealForumUpdater.update_forum! }
safely { RateLimit.prune! }
safely { regenerate_post_counts! }
end
def daily
safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all }
safely { PostDisapproval.prune! }
safely { TokenBucket.prune! }
safely { BulkUpdateRequestPruner.warn_old }
safely { BulkUpdateRequestPruner.reject_expired }
safely { Ban.prune! }

52
app/logical/rate_limiter.rb Normal file
View File

@@ -0,0 +1,52 @@
class RateLimiter
class RateLimitError < StandardError; end
attr_reader :action, :keys, :cost, :rate, :burst
def initialize(action, keys = ["*"], cost: 1, rate: 1, burst: 1)
@action = action
@keys = keys
@cost = cost
@rate = rate
@burst = burst
end
def self.for_action(controller_name, action_name, user, ip_addr)
action = "#{controller_name}:#{action_name}"
keys = [(user.cache_key unless user.is_anonymous?), "ip/#{ip_addr.to_s}"].compact
case action
when "users:create"
rate, burst = 1.0/5.minutes, 10
when "emails:update", "sessions:create", "moderation_reports:create"
rate, burst = 1.0/1.minute, 10
when "dmails:create", "comments:create", "forum_posts:create", "forum_topics:create"
rate, burst = 1.0/1.minute, 50
when "comment_votes:create", "comment_votes:destroy", "post_votes:create",
"post_votes:destroy", "favorites:create", "favorites:destroy", "post_disapprovals:create"
rate, burst = 1.0/1.second, 200
else
rate = user.api_regen_multiplier
burst = 200
end
RateLimiter.new(action, keys, rate: rate, burst: burst)
end
def limit!
raise RateLimitError if limited?
end
def limited?
rate_limits.any?(&:limited?)
end
def as_json(options = {})
hash = rate_limits.map { |limit| [limit.key, limit.points] }.to_h
super(options).except("keys", "rate_limits").merge(limits: hash)
end
def rate_limits
@rate_limits ||= RateLimit.create_or_update!(action: action, keys: keys, cost: cost, rate: rate, burst: burst)
end
end

14
app/logical/sources/strategies.rb
View File

@@ -3,17 +3,17 @@ module Sources
def self.all
[
Strategies::Pixiv,
Strategies::Fanbox,
Strategies::NicoSeiga,
Strategies::Twitter,
Strategies::Tumblr,
Strategies::NicoSeiga,
Strategies::Stash, # must come before DeviantArt
Strategies::DeviantArt,
Strategies::Tumblr,
Strategies::ArtStation,
Strategies::Nijie,
Strategies::Mastodon,
Strategies::Moebooru,
Strategies::Nijie,
Strategies::ArtStation,
Strategies::HentaiFoundry,
Strategies::Fanbox,
Strategies::Mastodon,
Strategies::Weibo,
Strategies::Newgrounds,
Strategies::Skeb
@@ -21,7 +21,7 @@ module Sources
end
def self.find(url, referer = nil, default: Strategies::Null)
strategy = all.map { |strategy| strategy.new(url, referer) }.detect(&:match?)
strategy = all.lazy.map { |s| s.new(url, referer) }.detect(&:match?)
strategy || default&.new(url, referer)
end

32
app/logical/sources/strategies/base.rb
View File

@@ -64,6 +64,10 @@ module Sources
# XXX should go in dedicated strategies.
case host
when /amazon\.(com|jp|co\.jp)\z/i
"Amazon"
when /ask\.fm\z/i
"Ask.fm"
when /bcy\.net\z/i
"BCY"
when /booth\.pm\z/i
@@ -72,6 +76,10 @@ module Sources
"Circle.ms"
when /dlsite\.(com|net)\z/i
"DLSite"
when /doujinshi\.mugimugi\.org\z/i, /doujinshi\.org\z/i
"Doujinshi.org"
when /erogamescape\.dyndns\.org\z/i
"Erogamescape"
when /facebook\.com\z/i
"Facebook"
when /fantia\.jp\z/i
@@ -82,18 +90,42 @@ module Sources
"Gumroad"
when /instagram\.com\z/i
"Instagram"
when /ko-fi\.com\z/i
"Ko-fi"
when /livedoor\.(jp|com)\z/i
"Livedoor"
when /lofter\.com\z/i
"Lofter"
when /mangaupdates\.com\z/i
"Mangaupdates"
when /melonbooks\.co\.jp\z/i
"Melonbooks"
when /mihuashi\.com\z/i
"Mihuashi"
when /mixi\.jp\z/i
"Mixi.jp"
when /patreon\.com\z/i
"Patreon"
when /piapro\.jp\z/i
"Piapro.jp"
when /picarto\.tv\z/i
"Picarto"
when /privatter\.net\z/i
"Privatter"
when /sakura\.ne\.jp\z/i
"Sakura.ne.jp"
when /stickam\.jp\z/i
"Stickam"
when /skeb\.jp\z/i
"Skeb"
when /tinami\.com\z/i
"Tinami"
when /toranoana\.(jp|shop)\z/i
"Toranoana"
when /twitch\.tv\z/i
"Twitch"
when /wikipedia\.org\z/i
"Wikipedia"
when /youtube\.com\z/i
"Youtube"
else

7
app/logical/sources/strategies/mastodon.rb
View File

@@ -17,7 +17,7 @@
module Sources::Strategies
class Mastodon < Base
HOST = %r{\Ahttps?://(?:www\.)?(?<domain>pawoo\.net|baraag\.net)}i
IMAGE = %r{\Ahttps?://(?:img\.pawoo\.net|baraag\.net)/media_attachments/files/(\d+/\d+/\d+)}
IMAGE = %r{\Ahttps?://(?:img\.pawoo\.net|baraag\.net(?:/system(?:/cache)?)?)/media_attachments/files/((?:\d+/)+\d+)}
NAMED_PROFILE = %r{#{HOST}/@(?<artist_name>\w+)}i
ID_PROFILE = %r{#{HOST}/web/accounts/(?<account_id>\d+)}
@@ -35,6 +35,7 @@ module Sources::Strategies
def file_host
case site_name
when "pawoo.net" then "img.pawoo.net"
when "baraag.net" then "baraag.net/system"
else site_name
end
end
@@ -85,7 +86,7 @@ module Sources::Strategies
end
def artist_name_from_url
url[NAMED_PROFILE, :artist_name]
urls.map { |url| url[NAMED_PROFILE, :artist_name] }.compact.first
end
def other_names
@@ -93,7 +94,7 @@ module Sources::Strategies
end
def account_id
url[ID_PROFILE, :account_id] || api_response.account_id
urls.map { |url| url[ID_PROFILE, :account_id] }.compact.first || api_response.account_id
end
def status_id_from_url

67
app/logical/sources/strategies/nijie.rb
View File

@@ -212,34 +212,57 @@ module Sources
return nil if page_url.blank? || client.blank?
response = client.cache(1.minute).get(page_url)
return nil unless response.status == 200
response&.parse
if response.status != 200 || response.parse.search("#login_illust").present?
clear_cached_session_cookie!
else
response.parse
end
end
memoize :page
def client
nijie = http.timeout(60).use(retriable: { max_retries: 20 })
cookie = Cache.get("nijie-session-cookie", 1.week) do
login_page = nijie.get("https://nijie.info/login.php").parse
form = {
email: Danbooru.config.nijie_login,
password: Danbooru.config.nijie_password,
url: login_page.at("input[name='url']")["value"],
save: "on",
ticket: ""
}
response = nijie.post("https://nijie.info/login_int.php", form: form)
DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200
return nil unless response.status == 200
response.cookies.select { |c| c.name == "NIJIEIJIEID" }.compact.first
end
nijie.cookies(NIJIEIJIEID: cookie, R18: 1)
return nil if cached_session_cookie.nil?
http.cookies(NIJIEIJIEID: cached_session_cookie, R18: 1)
end
memoize :client
def http
super.timeout(60).use(retriable: { max_retries: 20 })
end
def cached_session_cookie
Cache.get("nijie-session-cookie", 60.minutes, skip_nil: true) do
session_cookie
end
end
def clear_cached_session_cookie!
flush_cache # clear memoized session cookie
Cache.delete("nijie-session-cookie")
end
def session_cookie
login_page = http.get("https://nijie.info/login.php").parse
form = {
email: Danbooru.config.nijie_login,
password: Danbooru.config.nijie_password,
url: login_page.at("input[name='url']")["value"],
save: "on",
ticket: ""
}
response = http.post("https://nijie.info/login_int.php", form: form)
if response.status == 200
response.cookies.select { |c| c.name == "NIJIEIJIEID" }.compact.first
else
DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})"
nil
end
end
memoize :client, :cached_session_cookie
end
end
end

1
app/logical/sources/strategies/twitter.rb
View File

@@ -30,6 +30,7 @@ module Sources::Strategies
/(?<!\A)誕生祭(?:\d*)\z/,
/(?<!\A)版もうひとつの深夜の真剣お絵描き60分一本勝負(?:_\d+)?\z/,
/(?<!\A)版深夜の真剣お絵描き60分一本勝負(?:_\d+)?\z/,
/(?<!\A)版深夜の真剣お絵かき60分一本勝負(?:_\d+)?\z/,
/(?<!\A)深夜の真剣お絵描き60分一本勝負(?:_\d+)?\z/,
/(?<!\A)版深夜のお絵描き60分一本勝負(?:_\d+)?\z/,
/(?<!\A)版真剣お絵描き60分一本勝(?:_\d+)?\z/,

2
app/logical/tag_name_validator.rb
View File

@@ -10,7 +10,7 @@ class TagNameValidator < ActiveModel::EachValidator
case value
when /\A_*\z/
record.errors.add(attribute, "'#{value}' cannot be blank")
record.errors.add(attribute, "cannot be blank")
when /\*/
record.errors.add(attribute, "'#{value}' cannot contain asterisks ('*')")
when /,/

1
app/logical/user_promotion.rb
View File

@@ -18,7 +18,6 @@ class UserPromotion
user.level = new_level
user.can_upload_free = can_upload_free unless can_upload_free.nil?
user.can_approve_posts = can_approve_posts unless can_approve_posts.nil?
user.inviter = promoter
create_user_feedback
create_dmail