diff --git a/app/logical/deviant_art_api_client.rb b/app/logical/deviant_art_api_client.rb
index 3b8c35734..07bcf50a2 100644
--- a/app/logical/deviant_art_api_client.rb
+++ b/app/logical/deviant_art_api_client.rb
@@ -1,59 +1,73 @@
-# https://github.com/danbooru/danbooru/issues/4144
+# Authentication is via OAuth2 with the client credentials grant. Register a
+# new app at https://www.deviantart.com/developers/ to obtain a client_id and
+# client_secret. The app doesn't need to be published.
#
# API requests must send a user agent and must use gzip compression, otherwise
# 403 errors will be returned.
+#
+# API calls operate on UUIDs. The deviation ID in the URL is not the UUID. UUIDs
+# are obtained by scraping the HTML page for the element.
+#
+# * https://www.deviantart.com/developers/
+# * https://www.deviantart.com/developers/authentication
+# * https://www.deviantart.com/developers/errors
+# * https://www.deviantart.com/developers/http/v1/20160316
-DeviantArtApiClient = Struct.new(:deviation_id) do
- extend Memoist
+class DeviantArtApiClient
+ class Error < StandardError; end
+ BASE_URL = "https://www.deviantart.com/api/v1/oauth2"
- def extended_fetch
- params = { deviationid: deviation_id, type: "art", include_session: false }
- get("https://www.deviantart.com/_napi/da-deviation/shared_api/deviation/extended_fetch", params: params)
+ attr_reader :client_id, :client_secret, :httparty_options
+
+ def initialize(client_id, client_secret, httparty_options = {})
+ @client_id, @client_secret, @httparty_options = client_id, client_secret, httparty_options
end
- def extended_fetch_json
- JSON.parse(extended_fetch.body).with_indifferent_access
+ # https://www.deviantart.com/developers/http/v1/20160316/deviation_single/bcc296bdf3b5e40636825a942a514816
+ def deviation(uuid)
+ request("/deviation/#{uuid}")
end
- def download_url
- url = extended_fetch_json.dig(:deviation, :extended, :download, :url)
- response = get(url)
- response.headers[:location]
+ # https://www.deviantart.com/developers/http/v1/20160316/deviation_download/bed6982b88949bdb08b52cd6763fcafd
+ def download(uuid, mature_content: "1")
+ request("/deviation/download/#{uuid}", mature_content: mature_content)
end
- def get(url, retries: 1, **options)
- response = http.cookies(cookies).get(url, **options)
+ # https://www.deviantart.com/developers/http/v1/20160316/deviation_metadata/7824fc14d6fba6acbacca1cf38c24158
+ def metadata(*uuids, mature_content: "1", ext_submission: "1", ext_camera: "1", ext_stats: "1")
+ params = {
+ deviationids: uuids.flatten,
+ mature_content: mature_content,
+ ext_submission: ext_submission,
+ ext_camera: ext_camera,
+ ext_stats: ext_stats,
+ }
- new_cookies = response.cookies.cookies.map { |cookie| { cookie.name => cookie.value } }.reduce(&:merge)
- new_cookies = new_cookies.slice(:userinfo, :auth, :authsecure)
- if new_cookies.present?
- DanbooruLogger.info("DeviantArt: updating cookies", url: url, new_cookies: new_cookies, old_cookies: cookies)
- self.cookies = new_cookies
- end
-
- # If the old auth cookie expired we may get a 404 with a new auth cookie
- # set. Try again with the new cookie.
- if response.code == 404 && retries > 0
- DanbooruLogger.info("DeviantArt: retrying", url: url, cookies: cookies)
- response = get(url, retries: retries - 1, **options)
- end
-
- response
+ request("/deviation/metadata", **params)
end
- def cookies
- Cache.get("deviantart_cookies", 10.years.to_i) do
- JSON.parse(Danbooru.config.deviantart_cookies)
- end
+ def request(url, **params)
+ options = {
+ base_uri: BASE_URL,
+ params: { access_token: access_token.token, **params },
+ headers: { "Accept-Encoding" => "gzip" },
+ format: :plain,
+ }
+
+ body, code = HTTParty.get(url, **options)
+ JSON.parse(Zlib.gunzip(body), symbolize_names: true)
end
- def cookies=(new_cookies)
- Cache.put("deviantart_cookies", new_cookies, 10.years.to_i)
+ def oauth
+ OAuth2::Client.new(client_id, client_secret, site: "https://www.deviantart.com", token_url: "/oauth2/token")
end
- def http
- HTTP.use(:auto_inflate).headers(Danbooru.config.http_headers.merge("Accept-Encoding" => "gzip"))
+ def access_token
+ @access_token = oauth.client_credentials.get_token if @access_token.nil? || @access_token.expired?
+ @access_token
end
- memoize :extended_fetch, :extended_fetch_json, :download_url
+ def access_token=(hash)
+ @access_token = OAuth2::AccessToken.from_hash(oauth, hash)
+ end
end
diff --git a/app/logical/sources/strategies.rb b/app/logical/sources/strategies.rb
index 2c0982f2c..1c4f03d62 100644
--- a/app/logical/sources/strategies.rb
+++ b/app/logical/sources/strategies.rb
@@ -5,6 +5,7 @@ module Sources
Strategies::Pixiv,
Strategies::NicoSeiga,
Strategies::Twitter,
+ Strategies::Stash, # must come before DeviantArt
Strategies::DeviantArt,
Strategies::Tumblr,
Strategies::ArtStation,
diff --git a/app/logical/sources/strategies/deviant_art.rb b/app/logical/sources/strategies/deviant_art.rb
index f8e3306e9..1557586af 100644
--- a/app/logical/sources/strategies/deviant_art.rb
+++ b/app/logical/sources/strategies/deviant_art.rb
@@ -79,45 +79,36 @@ module Sources
end
def image_urls
- [image_url]
- end
-
- def image_url
# work is private, deleted, or the url didn't contain a deviation id; use image url as given by user.
if api_deviation.blank?
- url
- elsif api_deviation[:isDownloadable]
- api_client.download_url
+ [url]
+ elsif api_deviation[:is_downloadable]
+ src = api_download[:src]
+ src.sub!(%r!\Ahttps?://s3\.amazonaws\.com/!i, "https://")
+ src.sub!(/\?.*\z/, "") # strip s3 query params
+ src.sub!(%r!\Ahttps://origin-orig\.deviantart\.net!, "http://origin-orig.deviantart.net") # https://origin-orig.devianart.net doesn't work
+ [src]
+ elsif api_deviation.present?
+ src = api_deviation.dig(:content, :src)
+ if deviation_id && deviation_id.to_i <= 790677560 && src =~ /^https:\/\/images-wixmp-/
+ src = src.sub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
+ src = src.sub(%r!/v1/(fit|fill)/.*\z!i, "")
+ end
+ src = src.sub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
+ src = src.sub(%r!q_\d+!, "q_100")
+ [src]
else
- media = api_deviation[:media]
- token = media[:token].first
- fullview = media[:types].find { |data| data[:t] == "fullview" && data[:c].present? }
-
- if fullview.present?
- op = fullview[:c].gsub('', media[:prettyName])
- src = "#{media[:baseUri]}/#{op}?token=#{token}"
- else
- src = "#{media[:baseUri]}?token=#{token}"
- end
-
- if deviation_id && deviation_id.to_i <= 790677560 && src =~ /\Ahttps:\/\/images-wixmp-/i
- src = src.gsub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
- src = src.gsub(%r!/v1/(fit|fill)/.*\z!i, "")
- end
-
- src = src.gsub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
- src = src.gsub(%r!q_\d+,strp!, "q_100")
- src
+ raise "Couldn't find image url" # this should never happen
end
end
def page_url
- if api_deviation[:url].present?
+ if api_deviation.present?
api_deviation[:url]
- elsif deviation_id.present?
- page_url_from_image_url
+ elsif api_url.present?
+ api_url
else
- nil
+ ""
end
end
@@ -134,7 +125,7 @@ module Sources
end
def profile_url
- return nil if artist_name.blank?
+ return "" if artist_name.blank?
"https://www.deviantart.com/#{artist_name.downcase}"
end
@@ -143,20 +134,19 @@ module Sources
def artist_name
if artist_name_from_url.present?
artist_name_from_url
- elsif api_deviation.dig(:author, :username).present?
+ elsif api_metadata.present?
api_metadata.dig(:author, :username)
else
- nil
+ ""
end
end
def artist_commentary_title
- api_deviation[:title]
+ api_metadata[:title]
end
def artist_commentary_desc
- return nil unless api_deviation.dig(:extended, :description).present?
- api_deviation.dig(:extended, :description)
+ api_metadata[:description]
end
def normalized_for_artist_finder?
@@ -172,10 +162,12 @@ module Sources
end
def tags
- return [] unless api_deviation.dig(:extended, :tags).present?
+ if api_metadata.blank?
+ return []
+ end
- api_deviation.dig(:extended, :tags).map do |tag|
- [tag[:name], tag[:url]]
+ api_metadata[:tags].map do |tag|
+ [tag[:tag_name], "https://www.deviantart.com/tag/#{tag[:tag_name]}"]
end
end
@@ -209,6 +201,8 @@ module Sources
end.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
end
+ public
+
def self.deviation_id_from_url(url)
if url =~ ASSET
$~[:base36_deviation_id].try(:to_i, 36)
@@ -251,19 +245,81 @@ module Sources
self.class.title_from_url(url) || self.class.title_from_url(referer_url)
end
- def api_client
- @api_client ||= DeviantArtApiClient.new(deviation_id)
+ def api_url
+ return nil if deviation_id.blank?
+ "https://www.deviantart.com/deviation/#{deviation_id}"
end
- def api_deviation
- api_client.extended_fetch_json[:deviation] || {}
+ def page
+ return nil if api_url.blank?
+
+ options = Danbooru.config.httparty_options.deep_merge(
+ format: :plain,
+ headers: { "Accept-Encoding" => "gzip" }
+ )
+ resp = HTTParty.get(api_url, **options)
+
+ if resp.success?
+ body = Zlib.gunzip(resp.body)
+ Nokogiri::HTML(body)
+ # the work was deleted
+ elsif resp.code == 404
+ nil
+ else
+ raise HTTParty::ResponseError.new(resp)
+ end
end
+ memoize :page
+
+ # Scrape UUID from
+ # For hidden or deleted works the UUID will be nil.
+ def uuid
+ return nil if page.nil?
+ meta = page.search('meta[property="da:appurl"]').first
+ return nil if meta.nil?
+
+ appurl = meta["content"]
+ uuid = appurl[%r!\ADeviantArt://deviation/(.*)\z!, 1]
+ uuid
+ end
+ memoize :uuid
+
+ def api_client
+ api_client = DeviantArtApiClient.new(
+ Danbooru.config.deviantart_client_id,
+ Danbooru.config.deviantart_client_secret,
+ Danbooru.config.httparty_options
+ )
+ api_client.access_token = Cache.get("da-access-token", 55.minutes) do
+ api_client.access_token.to_hash
+ end
+ api_client
+ end
+ memoize :api_client
+
+ def api_deviation
+ return {} if uuid.nil?
+ api_client.deviation(uuid)
+ end
+ memoize :api_deviation
+
+ def api_metadata
+ return {} if uuid.nil?
+ api_client.metadata(uuid)[:metadata].first
+ end
+ memoize :api_metadata
+
+ def api_download
+ return {} if uuid.nil?
+ api_client.download(uuid)
+ end
+ memoize :api_download
def api_response
{
- code: api_client.extended_fetch.code,
- headers: api_client.extended_fetch.headers.to_h,
- body: api_client.extended_fetch_json
+ deviation: api_deviation,
+ metadata: api_metadata,
+ download: api_download,
}
end
end
diff --git a/app/logical/sources/strategies/stash.rb b/app/logical/sources/strategies/stash.rb
new file mode 100644
index 000000000..ebb79255a
--- /dev/null
+++ b/app/logical/sources/strategies/stash.rb
@@ -0,0 +1,55 @@
+# Page URLs:
+# * https://sta.sh/0wxs31o7nn2 (single image)
+# * https://sta.sh/21leo8mz87ue (folder)
+#
+# Image URLs:
+# * https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png
+#
+# Ref:
+# * https://github.com/danbooru/danbooru/issues/3877
+# * https://www.deviantartsupport.com/en/article/what-is-stash-3391708
+# * https://www.deviantart.com/developers/http/v1/20160316/stash_item/4662dd8b10e336486ea9a0b14da62b74
+#
+module Sources
+ module Strategies
+ class Stash < DeviantArt
+ STASH = %r{\Ahttps?://sta\.sh/(?[0-9a-zA-Z]+)}i
+
+ def domains
+ ["deviantart.net", "sta.sh"]
+ end
+
+ def match?
+ parsed_urls.map(&:domain).any?("sta.sh")
+ end
+
+ def site_name
+ "Sta.sh"
+ end
+
+ def canonical_url
+ page_url
+ end
+
+ def page_url
+ "https://sta.sh/#{stash_id}"
+ end
+
+ def api_url
+ page_url
+ end
+
+ def self.stash_id_from_url(url)
+ if url =~ STASH
+ $~[:post_id].downcase
+ else
+ nil
+ end
+ end
+
+ def stash_id
+ [url, referer_url].map{ |x| self.class.stash_id_from_url(x) }.compact.first
+ end
+ end
+ end
+end
diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb
index 634dde958..2f4ced182 100644
--- a/config/danbooru_default_config.rb
+++ b/config/danbooru_default_config.rb
@@ -333,6 +333,15 @@ module Danbooru
nil
end
+ # Register at https://www.deviantart.com/developers/
+ def deviantart_client_id
+ nil
+ end
+
+ def deviantart_client_secret
+ nil
+ end
+
# http://tinysubversions.com/notes/mastodon-bot/
def pawoo_client_id
nil
diff --git a/test/unit/artist_test.rb b/test/unit/artist_test.rb
index 8f632b512..b9dc3a90c 100644
--- a/test/unit/artist_test.rb
+++ b/test/unit/artist_test.rb
@@ -205,6 +205,7 @@ class ArtistTest < ActiveSupport::TestCase
context "when finding deviantart artists" do
setup do
+ skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
FactoryBot.create(:artist, :name => "artgerm", :url_string => "http://artgerm.deviantart.com/")
FactoryBot.create(:artist, :name => "trixia", :url_string => "http://trixdraws.deviantart.com/")
end
diff --git a/test/unit/sources/deviant_art_test.rb b/test/unit/sources/deviant_art_test.rb
index bda6bef2b..dbc084106 100644
--- a/test/unit/sources/deviant_art_test.rb
+++ b/test/unit/sources/deviant_art_test.rb
@@ -2,6 +2,11 @@ require 'test_helper'
module Sources
class DeviantArtTest < ActiveSupport::TestCase
+ def setup
+ super
+ skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
+ end
+
context "A page url" do
setup do
@site = Sources::Strategies.find("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484")
diff --git a/test/unit/sources/stash_test.rb b/test/unit/sources/stash_test.rb
new file mode 100644
index 000000000..b005bb9c8
--- /dev/null
+++ b/test/unit/sources/stash_test.rb
@@ -0,0 +1,58 @@
+require 'test_helper'
+
+module Sources
+ class StashTest < ActiveSupport::TestCase
+ def setup
+ super
+ skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
+ end
+
+ context "A https://sta.sh/:id page url" do
+ should "work" do
+ @site = Sources::Strategies.find("https://sta.sh/0wxs31o7nn2")
+
+ assert_equal("noizave", @site.artist_name)
+ assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
+
+ assert_equal("A pepe", @site.artist_commentary_title)
+ assert_equal("This is a test.", @site.artist_commentary_desc)
+
+ assert_equal("https://sta.sh/0wxs31o7nn2", @site.page_url)
+ assert_equal("https://sta.sh/0wxs31o7nn2", @site.canonical_url)
+ assert_equal("http://origin-orig.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png", @site.image_url)
+ assert_equal(["http://origin-orig.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png"], @site.image_urls)
+ end
+ end
+
+ context "A https://orig00.deviantart.net/* image url" do
+ context "with a https://sta.sh/:id referer" do
+ should "work" do
+ @site = Sources::Strategies.find("https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png", "https://sta.sh/0wxs31o7nn2")
+
+ assert_equal("noizave", @site.artist_name)
+ assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
+
+ assert_equal("A pepe", @site.artist_commentary_title)
+ assert_equal("This is a test.", @site.artist_commentary_desc)
+
+ assert_equal("https://sta.sh/0wxs31o7nn2", @site.page_url)
+ assert_equal("https://sta.sh/0wxs31o7nn2", @site.canonical_url)
+ assert_equal("http://origin-orig.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png", @site.image_url)
+ assert_equal(["http://origin-orig.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png"], @site.image_urls)
+ end
+ end
+
+ context "without a referer" do
+ should "use the base deviantart strategy" do
+ @site = Sources::Strategies.find("https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png")
+
+ # if all we have is the image url, then we can't tell that this is really a sta.sh image.
+ assert_equal("Deviant Art", @site.site_name)
+
+ # this is the wrong page, but there's no way to know the correct sta.sh page without the referer.
+ assert_equal("https://www.deviantart.com/deviation/763305148", @site.page_url)
+ end
+ end
+ end
+ end
+end