jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix #4144: Deviantart Eclipse update broke strategy.

Browse Source
This commit is contained in:
evazion
2019-08-28 23:40:29 -05:00
parent eb94bf366b
commit eba6440b8b
10 changed files with 131 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
app/logical/deviant_art_api_client.rb
View File

@@ -1,73 +1,29 @@
# Authentication is via OAuth2 with the client credentials grant. Register a
# new app at https://www.deviantart.com/developers/ to obtain a client_id and
# client_secret. The app doesn't need to be published.
# https://github.com/r888888888/danbooru/issues/4144
#
# API requests must send a user agent and must use gzip compression, otherwise
# 403 errors will be returned.
#
# API calls operate on UUIDs. The deviation ID in the URL is not the UUID. UUIDs
# are obtained by scraping the HTML page for the <meta property="da:appurl"> element.
#
# * https://www.deviantart.com/developers/
# * https://www.deviantart.com/developers/authentication
# * https://www.deviantart.com/developers/errors
# * https://www.deviantart.com/developers/http/v1/20160316
class DeviantArtApiClient
class Error < StandardError; end
BASE_URL = "https://www.deviantart.com/api/v1/oauth2"
class DeviantArtApiClient < Struct.new(:deviation_id)
extend Memoist
attr_reader :client_id, :client_secret, :httparty_options
def initialize(client_id, client_secret, httparty_options = {})
@client_id, @client_secret, @httparty_options = client_id, client_secret, httparty_options
def extended_fetch
params = { deviationid: deviation_id, type: "art", include_session: false }
http.get("https://www.deviantart.com/_napi/da-deviation/shared_api/deviation/extended_fetch", params: params)
end
# https://www.deviantart.com/developers/http/v1/20160316/deviation_single/bcc296bdf3b5e40636825a942a514816
def deviation(uuid)
request("/deviation/#{uuid}")
def extended_fetch_json
JSON.parse(extended_fetch.body).with_indifferent_access
end
# https://www.deviantart.com/developers/http/v1/20160316/deviation_download/bed6982b88949bdb08b52cd6763fcafd
def download(uuid, mature_content: "1")
request("/deviation/download/#{uuid}", mature_content: mature_content)
def download_url
url = extended_fetch_json.dig(:deviation, :extended, :download, :url)
response = http.cookies(extended_fetch.cookies).get(url)
response.headers[:location]
end
# https://www.deviantart.com/developers/http/v1/20160316/deviation_metadata/7824fc14d6fba6acbacca1cf38c24158
def metadata(*uuids, mature_content: "1", ext_submission: "1", ext_camera: "1", ext_stats: "1")
params = {
deviationids: uuids.flatten,
mature_content: mature_content,
ext_submission: ext_submission,
ext_camera: ext_camera,
ext_stats: ext_stats,
}
request("/deviation/metadata", params)
def http
HTTP.use(:auto_inflate).headers(Danbooru.config.http_headers.merge("Accept-Encoding" => "gzip"))
end
def request(url, **params)
options = {
base_uri: BASE_URL,
params: { access_token: access_token.token, **params },
headers: { "Accept-Encoding" => "gzip" },
format: :plain,
}
body, code = HttpartyCache.get(url, **options)
JSON.parse(Zlib.gunzip(body), symbolize_names: true)
end
def oauth
OAuth2::Client.new(client_id, client_secret, site: "https://www.deviantart.com", token_url: "/oauth2/token")
end
def access_token
@access_token = oauth.client_credentials.get_token if @access_token.nil? || @access_token.expired?
@access_token
end
def access_token=(hash)
@access_token = OAuth2::AccessToken.from_hash(oauth, hash)
end
memoize :extended_fetch, :extended_fetch_json, :download_url
end

13
app/logical/sources/strategies.rb
View File

@@ -2,14 +2,13 @@ module Sources
module Strategies
def self.all
return [
Strategies::Pixiv,
Strategies::Pixiv,
Strategies::NicoSeiga,
Strategies::Twitter,
Strategies::Stash, # must come before DeviantArt
Strategies::DeviantArt,
Strategies::Tumblr,
Strategies::ArtStation,
Strategies::Nijie,
Strategies::Twitter,
Strategies::DeviantArt,
Strategies::Tumblr,
Strategies::ArtStation,
Strategies::Nijie,
Strategies::Pawoo,
Strategies::Moebooru,
]

2
app/logical/sources/strategies/base.rb
View File

@@ -178,7 +178,7 @@ module Sources
end
def artists
Artist.find_artists(normalize_for_artist_finder)
Artist.find_artists(normalize_for_artist_finder.to_s)
end
# A new artist entry with suggested defaults for when the artist doesn't

138
app/logical/sources/strategies/deviant_art.rb
View File

@@ -57,6 +57,8 @@ module Sources
PATH_PROFILE = %r{\Ahttps?://(www\.)?deviantart\.com/#{ARTIST}/?\z}i
SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i
attr_accessor :api_client
def domains
["deviantart.net", "deviantart.com"]
end
@@ -79,38 +81,37 @@ module Sources
end
def image_urls
[image_url]
end
def image_url
# work is private, deleted, or the url didn't contain a deviation id; use image url as given by user.
if api_deviation.blank?
[url]
# work is downloadable
elsif api_deviation[:is_downloadable]
src = api_download[:src]
src.sub!(%r!\Ahttps?://s3\.amazonaws\.com/!i, "https://")
src.sub!(/\?.*\z/, "") # strip s3 query params
src.sub!(%r!\Ahttps://origin-orig\.deviantart\.net!, "http://origin-orig.deviantart.net") # https://origin-orig.devianart.net doesn't work
[src]
# work isn't downloadable, or download size is same as regular size.
elsif api_deviation.present?
src = api_deviation.dig(:content, :src)
if deviation_id && deviation_id.to_i <= 790677560 && src =~ /^https:\/\/images-wixmp-/
src = src.sub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
src = src.sub(%r!/v1/(fit|fill)/.*\z!i, "")
end
src = src.sub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
src = src.sub(%r!q_\d+!, "q_100")
[src]
url
elsif api_deviation[:isDownloadable]
api_client.download_url
else
raise "Couldn't find image url" # this should never happen
file = api_deviation[:files].find { |data| data[:type] == "fullview" }
src = file[:src]
if deviation_id && deviation_id.to_i <= 790677560 && src =~ /\Ahttps:\/\/images-wixmp-/i
src = src.gsub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
src = src.gsub(%r!/v1/(fit|fill)/.*\z!i, "")
end
src = src.gsub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
src = src.gsub(%r!q_\d+,strp!, "q_100")
src
end
end
def page_url
if api_deviation.present?
if api_deviation[:url].present?
api_deviation[:url]
elsif api_url.present?
api_url
elsif deviation_id.present?
page_url_from_image_url
else
""
nil
end
end
@@ -127,7 +128,7 @@ module Sources
end
def profile_url
return "" if artist_name.blank?
return nil if artist_name.blank?
"https://www.deviantart.com/#{artist_name.downcase}"
end
@@ -136,19 +137,20 @@ module Sources
def artist_name
if artist_name_from_url.present?
artist_name_from_url
elsif api_metadata.present?
elsif api_deviation.dig(:author, :username).present?
api_metadata.dig(:author, :username)
else
""
nil
end
end
def artist_commentary_title
api_metadata[:title]
api_deviation[:title]
end
def artist_commentary_desc
api_metadata[:description]
return nil unless api_deviation.dig(:extended, :description).present?
api_deviation.dig(:extended, :description)
end
def normalized_for_artist_finder?
@@ -164,12 +166,10 @@ module Sources
end
def tags
if api_metadata.blank?
return []
end
return [] unless api_deviation.dig(:extended, :tags).present?
api_metadata[:tags].map do |tag|
[tag[:tag_name], "https://www.deviantart.com/tag/#{tag[:tag_name]}"]
api_deviation.dig(:extended, :tags).map do |tag|
[tag[:name], tag[:url]]
end
end
@@ -247,81 +247,19 @@ module Sources
self.class.title_from_url(url) || self.class.title_from_url(referer_url)
end
def api_url
return nil if deviation_id.blank?
"https://www.deviantart.com/deviation/#{deviation_id}"
end
def page
return nil if api_url.blank?
options = Danbooru.config.httparty_options.deep_merge(
format: :plain,
headers: { "Accept-Encoding" => "gzip" }
)
resp = HTTParty.get(api_url, **options)
if resp.success?
body = Zlib.gunzip(resp.body)
Nokogiri::HTML(body)
# the work was deleted
elsif resp.code == 404
nil
else
raise HTTParty::ResponseError.new(resp)
end
end
memoize :page
# Scrape UUID from <meta property="da:appurl" content="DeviantArt://deviation/12F08C5D-A3A4-338C-2F1A-7E4E268C0E8B">
# For hidden or deleted works the UUID will be nil.
def uuid
return nil if page.nil?
meta = page.search('meta[property="da:appurl"]').first
return nil if meta.nil?
appurl = meta["content"]
uuid = appurl[%r!\ADeviantArt://deviation/(.*)\z!, 1]
uuid
end
memoize :uuid
def api_client
api_client = DeviantArtApiClient.new(
Danbooru.config.deviantart_client_id,
Danbooru.config.deviantart_client_secret,
Danbooru.config.httparty_options
)
api_client.access_token = Cache.get("da-access-token", 55.minutes) do
api_client.access_token.to_hash
end
api_client
@api_client ||= DeviantArtApiClient.new(deviation_id)
end
memoize :api_client
def api_deviation
return {} if uuid.nil?
api_client.deviation(uuid)
api_client.extended_fetch_json[:deviation] || {}
end
memoize :api_deviation
def api_metadata
return {} if uuid.nil?
api_client.metadata(uuid)[:metadata].first
end
memoize :api_metadata
def api_download
return {} if uuid.nil?
api_client.download(uuid)
end
memoize :api_download
def api_response
{
deviation: api_deviation,
metadata: api_metadata,
download: api_download,
code: api_client.extended_fetch.code,
headers: api_client.extended_fetch.headers.to_h,
body: api_client.extended_fetch_json
}
end
end

55
app/logical/sources/strategies/stash.rb
View File

@@ -1,55 +0,0 @@
# Page URLs:
# * https://sta.sh/0wxs31o7nn2 (single image)
# * https://sta.sh/21leo8mz87ue (folder)
#
# Image URLs:
# * https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png
#
# Ref:
# * https://github.com/r888888888/danbooru/issues/3877
# * https://www.deviantartsupport.com/en/article/what-is-stash-3391708
# * https://www.deviantart.com/developers/http/v1/20160316/stash_item/4662dd8b10e336486ea9a0b14da62b74
#
module Sources
module Strategies
class Stash < DeviantArt
STASH = %r{\Ahttps?://sta\.sh/(?<post_id>[0-9a-zA-Z]+)}i
def domains
["deviantart.net", "sta.sh"]
end
def match?
parsed_urls.map(&:domain).any?("sta.sh")
end
def site_name
"Sta.sh"
end
def canonical_url
page_url
end
def page_url
"https://sta.sh/#{stash_id}"
end
def api_url
page_url
end
def self.stash_id_from_url(url)
if url =~ STASH
$~[:post_id].downcase
else
nil
end
end
def stash_id
[url, referer_url].map{ |x| self.class.stash_id_from_url(x) }.compact.first
end
end
end
end