jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix #4144: Deviantart Eclipse update broke strategy.

Browse Source
This commit is contained in:
evazion
2019-08-28 23:40:29 -05:00
parent eb94bf366b
commit eba6440b8b
10 changed files with 131 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
app/logical/sources/strategies.rb
View File

@@ -2,14 +2,13 @@ module Sources
module Strategies
def self.all
return [
Strategies::Pixiv,
Strategies::Pixiv,
Strategies::NicoSeiga,
Strategies::Twitter,
Strategies::Stash, # must come before DeviantArt
Strategies::DeviantArt,
Strategies::Tumblr,
Strategies::ArtStation,
Strategies::Nijie,
Strategies::Twitter,
Strategies::DeviantArt,
Strategies::Tumblr,
Strategies::ArtStation,
Strategies::Nijie,
Strategies::Pawoo,
Strategies::Moebooru,
]

2
app/logical/sources/strategies/base.rb
View File

@@ -178,7 +178,7 @@ module Sources
end
def artists
Artist.find_artists(normalize_for_artist_finder)
Artist.find_artists(normalize_for_artist_finder.to_s)
end
# A new artist entry with suggested defaults for when the artist doesn't

138
app/logical/sources/strategies/deviant_art.rb
View File

@@ -57,6 +57,8 @@ module Sources
PATH_PROFILE = %r{\Ahttps?://(www\.)?deviantart\.com/#{ARTIST}/?\z}i
SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i
attr_accessor :api_client
def domains
["deviantart.net", "deviantart.com"]
end
@@ -79,38 +81,37 @@ module Sources
end
def image_urls
[image_url]
end
def image_url
# work is private, deleted, or the url didn't contain a deviation id; use image url as given by user.
if api_deviation.blank?
[url]
# work is downloadable
elsif api_deviation[:is_downloadable]
src = api_download[:src]
src.sub!(%r!\Ahttps?://s3\.amazonaws\.com/!i, "https://")
src.sub!(/\?.*\z/, "") # strip s3 query params
src.sub!(%r!\Ahttps://origin-orig\.deviantart\.net!, "http://origin-orig.deviantart.net") # https://origin-orig.devianart.net doesn't work
[src]
# work isn't downloadable, or download size is same as regular size.
elsif api_deviation.present?
src = api_deviation.dig(:content, :src)
if deviation_id && deviation_id.to_i <= 790677560 && src =~ /^https:\/\/images-wixmp-/
src = src.sub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
src = src.sub(%r!/v1/(fit|fill)/.*\z!i, "")
end
src = src.sub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
src = src.sub(%r!q_\d+!, "q_100")
[src]
url
elsif api_deviation[:isDownloadable]
api_client.download_url
else
raise "Couldn't find image url" # this should never happen
file = api_deviation[:files].find { |data| data[:type] == "fullview" }
src = file[:src]
if deviation_id && deviation_id.to_i <= 790677560 && src =~ /\Ahttps:\/\/images-wixmp-/i
src = src.gsub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
src = src.gsub(%r!/v1/(fit|fill)/.*\z!i, "")
end
src = src.gsub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
src = src.gsub(%r!q_\d+,strp!, "q_100")
src
end
end
def page_url
if api_deviation.present?
if api_deviation[:url].present?
api_deviation[:url]
elsif api_url.present?
api_url
elsif deviation_id.present?
page_url_from_image_url
else
""
nil
end
end
@@ -127,7 +128,7 @@ module Sources
end
def profile_url
return "" if artist_name.blank?
return nil if artist_name.blank?
"https://www.deviantart.com/#{artist_name.downcase}"
end
@@ -136,19 +137,20 @@ module Sources
def artist_name
if artist_name_from_url.present?
artist_name_from_url
elsif api_metadata.present?
elsif api_deviation.dig(:author, :username).present?
api_metadata.dig(:author, :username)
else
""
nil
end
end
def artist_commentary_title
api_metadata[:title]
api_deviation[:title]
end
def artist_commentary_desc
api_metadata[:description]
return nil unless api_deviation.dig(:extended, :description).present?
api_deviation.dig(:extended, :description)
end
def normalized_for_artist_finder?
@@ -164,12 +166,10 @@ module Sources
end
def tags
if api_metadata.blank?
return []
end
return [] unless api_deviation.dig(:extended, :tags).present?
api_metadata[:tags].map do |tag|
[tag[:tag_name], "https://www.deviantart.com/tag/#{tag[:tag_name]}"]
api_deviation.dig(:extended, :tags).map do |tag|
[tag[:name], tag[:url]]
end
end
@@ -247,81 +247,19 @@ module Sources
self.class.title_from_url(url) || self.class.title_from_url(referer_url)
end
def api_url
return nil if deviation_id.blank?
"https://www.deviantart.com/deviation/#{deviation_id}"
end
def page
return nil if api_url.blank?
options = Danbooru.config.httparty_options.deep_merge(
format: :plain,
headers: { "Accept-Encoding" => "gzip" }
)
resp = HTTParty.get(api_url, **options)
if resp.success?
body = Zlib.gunzip(resp.body)
Nokogiri::HTML(body)
# the work was deleted
elsif resp.code == 404
nil
else
raise HTTParty::ResponseError.new(resp)
end
end
memoize :page
# Scrape UUID from <meta property="da:appurl" content="DeviantArt://deviation/12F08C5D-A3A4-338C-2F1A-7E4E268C0E8B">
# For hidden or deleted works the UUID will be nil.
def uuid
return nil if page.nil?
meta = page.search('meta[property="da:appurl"]').first
return nil if meta.nil?
appurl = meta["content"]
uuid = appurl[%r!\ADeviantArt://deviation/(.*)\z!, 1]
uuid
end
memoize :uuid
def api_client
api_client = DeviantArtApiClient.new(
Danbooru.config.deviantart_client_id,
Danbooru.config.deviantart_client_secret,
Danbooru.config.httparty_options
)
api_client.access_token = Cache.get("da-access-token", 55.minutes) do
api_client.access_token.to_hash
end
api_client
@api_client ||= DeviantArtApiClient.new(deviation_id)
end
memoize :api_client
def api_deviation
return {} if uuid.nil?
api_client.deviation(uuid)
api_client.extended_fetch_json[:deviation] || {}
end
memoize :api_deviation
def api_metadata
return {} if uuid.nil?
api_client.metadata(uuid)[:metadata].first
end
memoize :api_metadata
def api_download
return {} if uuid.nil?
api_client.download(uuid)
end
memoize :api_download
def api_response
{
deviation: api_deviation,
metadata: api_metadata,
download: api_download,
code: api_client.extended_fetch.code,
headers: api_client.extended_fetch.headers.to_h,
body: api_client.extended_fetch_json
}
end
end

55
app/logical/sources/strategies/stash.rb
View File

@@ -1,55 +0,0 @@
# Page URLs:
# * https://sta.sh/0wxs31o7nn2 (single image)
# * https://sta.sh/21leo8mz87ue (folder)
#
# Image URLs:
# * https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png
#
# Ref:
# * https://github.com/r888888888/danbooru/issues/3877
# * https://www.deviantartsupport.com/en/article/what-is-stash-3391708
# * https://www.deviantart.com/developers/http/v1/20160316/stash_item/4662dd8b10e336486ea9a0b14da62b74
#
module Sources
module Strategies
class Stash < DeviantArt
STASH = %r{\Ahttps?://sta\.sh/(?<post_id>[0-9a-zA-Z]+)}i
def domains
["deviantart.net", "sta.sh"]
end
def match?
parsed_urls.map(&:domain).any?("sta.sh")
end
def site_name
"Sta.sh"
end
def canonical_url
page_url
end
def page_url
"https://sta.sh/#{stash_id}"
end
def api_url
page_url
end
def self.stash_id_from_url(url)
if url =~ STASH
$~[:post_id].downcase
else
nil
end
end
def stash_id
[url, referer_url].map{ |x| self.class.stash_id_from_url(x) }.compact.first
end
end
end
end