Fix #4144: Deviantart Eclipse update broke strategy.

app/logical/deviant_art_api_client.rb

@@ -1,73 +1,29 @@
-# Authentication is via OAuth2 with the client credentials grant. Register a
+# https://github.com/r888888888/danbooru/issues/4144
-# new app at https://www.deviantart.com/developers/ to obtain a client_id and
-# client_secret. The app doesn't need to be published.
 #
 # API requests must send a user agent and must use gzip compression, otherwise
 # 403 errors will be returned.
-#
-# API calls operate on UUIDs. The deviation ID in the URL is not the UUID. UUIDs
-# are obtained by scraping the HTML page for the <meta property="da:appurl"> element.
-#
-# * https://www.deviantart.com/developers/
-# * https://www.deviantart.com/developers/authentication
-# * https://www.deviantart.com/developers/errors
-# * https://www.deviantart.com/developers/http/v1/20160316
 
-class DeviantArtApiClient
+class DeviantArtApiClient < Struct.new(:deviation_id)
-  class Error < StandardError; end
+  extend Memoist
-  BASE_URL = "https://www.deviantart.com/api/v1/oauth2"
 
-  attr_reader :client_id, :client_secret, :httparty_options
+  def extended_fetch
-
+    params = { deviationid: deviation_id, type: "art", include_session: false }
-  def initialize(client_id, client_secret, httparty_options = {})
+    http.get("https://www.deviantart.com/_napi/da-deviation/shared_api/deviation/extended_fetch", params: params)
-    @client_id, @client_secret, @httparty_options = client_id, client_secret, httparty_options
   end
 
-  # https://www.deviantart.com/developers/http/v1/20160316/deviation_single/bcc296bdf3b5e40636825a942a514816
+  def extended_fetch_json
-  def deviation(uuid)
+    JSON.parse(extended_fetch.body).with_indifferent_access
-    request("/deviation/#{uuid}")
   end
 
-  # https://www.deviantart.com/developers/http/v1/20160316/deviation_download/bed6982b88949bdb08b52cd6763fcafd
+  def download_url
-  def download(uuid, mature_content: "1")
+    url = extended_fetch_json.dig(:deviation, :extended, :download, :url)
-    request("/deviation/download/#{uuid}", mature_content: mature_content)
+    response = http.cookies(extended_fetch.cookies).get(url)
+    response.headers[:location]
   end
 
-  # https://www.deviantart.com/developers/http/v1/20160316/deviation_metadata/7824fc14d6fba6acbacca1cf38c24158
+  def http
-  def metadata(*uuids, mature_content: "1", ext_submission: "1", ext_camera: "1", ext_stats: "1")
+    HTTP.use(:auto_inflate).headers(Danbooru.config.http_headers.merge("Accept-Encoding" => "gzip"))
-    params = {
-      deviationids: uuids.flatten,
-      mature_content: mature_content,
-      ext_submission: ext_submission,
-      ext_camera: ext_camera,
-      ext_stats: ext_stats,
-    }
-
-    request("/deviation/metadata", params)
   end
 
-  def request(url, **params)
+  memoize :extended_fetch, :extended_fetch_json, :download_url
-    options = {
-      base_uri: BASE_URL,
-      params: { access_token: access_token.token, **params },
-      headers: { "Accept-Encoding" => "gzip" },
-      format: :plain,
-    }
-
-    body, code = HttpartyCache.get(url, **options)
-     JSON.parse(Zlib.gunzip(body), symbolize_names: true)
-  end
-
-  def oauth
-    OAuth2::Client.new(client_id, client_secret, site: "https://www.deviantart.com", token_url: "/oauth2/token")
-  end
-
-  def access_token
-    @access_token = oauth.client_credentials.get_token if @access_token.nil? || @access_token.expired?
-    @access_token
-  end
-
-  def access_token=(hash)
-    @access_token = OAuth2::AccessToken.from_hash(oauth, hash)
-  end
 end

app/logical/sources/strategies.rb

@@ -5,7 +5,6 @@ module Sources
         Strategies::Pixiv,
         Strategies::NicoSeiga,
         Strategies::Twitter,
-        Strategies::Stash, # must come before DeviantArt
         Strategies::DeviantArt,
         Strategies::Tumblr,
         Strategies::ArtStation,

app/logical/sources/strategies/base.rb

@@ -178,7 +178,7 @@ module Sources
       end
 
       def artists
-        Artist.find_artists(normalize_for_artist_finder)
+        Artist.find_artists(normalize_for_artist_finder.to_s)
       end
 
       # A new artist entry with suggested defaults for when the artist doesn't

app/logical/sources/strategies/deviant_art.rb

@@ -57,6 +57,8 @@ module Sources
       PATH_PROFILE = %r{\Ahttps?://(www\.)?deviantart\.com/#{ARTIST}/?\z}i
       SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i
 
+      attr_accessor :api_client
+
       def domains
         ["deviantart.net", "deviantart.com"]
       end
@@ -79,38 +81,37 @@ module Sources
       end
 
       def image_urls
+        [image_url]
+      end
+
+      def image_url
         # work is private, deleted, or the url didn't contain a deviation id; use image url as given by user.
         if api_deviation.blank?
-          [url]
+          url
-        # work is downloadable
+        elsif api_deviation[:isDownloadable]
-        elsif api_deviation[:is_downloadable]
+          api_client.download_url
-          src = api_download[:src]
-          src.sub!(%r!\Ahttps?://s3\.amazonaws\.com/!i, "https://")
-          src.sub!(/\?.*\z/, "") # strip s3 query params
-          src.sub!(%r!\Ahttps://origin-orig\.deviantart\.net!, "http://origin-orig.deviantart.net") # https://origin-orig.devianart.net doesn't work
-          [src]
-        # work isn't downloadable, or download size is same as regular size.
-        elsif api_deviation.present?
-          src = api_deviation.dig(:content, :src)
-          if deviation_id && deviation_id.to_i <= 790677560 && src =~ /^https:\/\/images-wixmp-/
-            src = src.sub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
-            src = src.sub(%r!/v1/(fit|fill)/.*\z!i, "")
-          end
-          src = src.sub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
-          src = src.sub(%r!q_\d+!, "q_100")
-          [src]
         else
-          raise "Couldn't find image url" # this should never happen
+          file = api_deviation[:files].find { |data| data[:type] == "fullview" }
+          src = file[:src]
+
+          if deviation_id && deviation_id.to_i <= 790677560 && src =~ /\Ahttps:\/\/images-wixmp-/i
+            src = src.gsub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
+            src = src.gsub(%r!/v1/(fit|fill)/.*\z!i, "")
+          end
+
+          src = src.gsub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
+          src = src.gsub(%r!q_\d+,strp!, "q_100")
+          src
         end
       end
 
       def page_url
-        if api_deviation.present?
+        if api_deviation[:url].present?
           api_deviation[:url]
-        elsif api_url.present?
+        elsif deviation_id.present?
-          api_url
+          page_url_from_image_url
         else
-          ""
+          nil
         end
       end
 
@@ -127,7 +128,7 @@ module Sources
       end
 
       def profile_url
-        return "" if artist_name.blank?
+        return nil if artist_name.blank?
         "https://www.deviantart.com/#{artist_name.downcase}"
       end
 
@@ -136,19 +137,20 @@ module Sources
       def artist_name
         if artist_name_from_url.present?
           artist_name_from_url
-        elsif api_metadata.present?
+        elsif api_deviation.dig(:author, :username).present?
           api_metadata.dig(:author, :username)
         else
-          ""
+          nil
         end
       end
 
       def artist_commentary_title
-        api_metadata[:title]
+        api_deviation[:title]
       end
 
       def artist_commentary_desc
-        api_metadata[:description]
+        return nil unless api_deviation.dig(:extended, :description).present?
+        api_deviation.dig(:extended, :description)
       end
 
       def normalized_for_artist_finder?
@@ -164,12 +166,10 @@ module Sources
       end
 
       def tags
-        if api_metadata.blank?
+        return [] unless api_deviation.dig(:extended, :tags).present?
-          return []
-        end
 
-        api_metadata[:tags].map do |tag|
+        api_deviation.dig(:extended, :tags).map do |tag|
-          [tag[:tag_name], "https://www.deviantart.com/tag/#{tag[:tag_name]}"]
+          [tag[:name], tag[:url]]
         end
       end
 
@@ -247,81 +247,19 @@ module Sources
         self.class.title_from_url(url) || self.class.title_from_url(referer_url)
       end
 
-      def api_url
-        return nil if deviation_id.blank?
-        "https://www.deviantart.com/deviation/#{deviation_id}"
-      end
-
-      def page
-        return nil if api_url.blank?
-
-        options = Danbooru.config.httparty_options.deep_merge(
-          format: :plain, 
-          headers: { "Accept-Encoding" => "gzip" }
-        )
-        resp = HTTParty.get(api_url, **options)
-
-        if resp.success?
-          body = Zlib.gunzip(resp.body)
-          Nokogiri::HTML(body)
-        # the work was deleted
-        elsif resp.code == 404
-          nil
-        else
-          raise HTTParty::ResponseError.new(resp)
-        end
-      end
-      memoize :page
-
-      # Scrape UUID from <meta property="da:appurl" content="DeviantArt://deviation/12F08C5D-A3A4-338C-2F1A-7E4E268C0E8B">
-      # For hidden or deleted works the UUID will be nil.
-      def uuid
-        return nil if page.nil?
-        meta = page.search('meta[property="da:appurl"]').first
-        return nil if meta.nil?
-
-        appurl = meta["content"]
-        uuid = appurl[%r!\ADeviantArt://deviation/(.*)\z!, 1]
-        uuid
-      end
-      memoize :uuid
-
       def api_client
-        api_client = DeviantArtApiClient.new(
+        @api_client ||= DeviantArtApiClient.new(deviation_id)
-          Danbooru.config.deviantart_client_id, 
-          Danbooru.config.deviantart_client_secret, 
-          Danbooru.config.httparty_options
-        )
-        api_client.access_token = Cache.get("da-access-token", 55.minutes) do
-          api_client.access_token.to_hash
-        end
-        api_client
       end
-      memoize :api_client
 
       def api_deviation
-        return {} if uuid.nil?
+        api_client.extended_fetch_json[:deviation] || {}
-        api_client.deviation(uuid)
       end
-      memoize :api_deviation
-
-      def api_metadata
-        return {} if uuid.nil?
-        api_client.metadata(uuid)[:metadata].first
-      end
-      memoize :api_metadata
-
-      def api_download
-        return {} if uuid.nil?
-        api_client.download(uuid)
-      end
-      memoize :api_download
 
       def api_response
         {
-          deviation: api_deviation,
+          code: api_client.extended_fetch.code,
-          metadata: api_metadata,
+          headers: api_client.extended_fetch.headers.to_h,
-          download: api_download,
+          body: api_client.extended_fetch_json
         }
       end
     end

app/logical/sources/strategies/stash.rb

@@ -1,55 +0,0 @@
-# Page URLs:
-# * https://sta.sh/0wxs31o7nn2 (single image)
-# * https://sta.sh/21leo8mz87ue (folder)
-#
-# Image URLs:
-# * https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png
-#
-# Ref:
-# * https://github.com/r888888888/danbooru/issues/3877
-# * https://www.deviantartsupport.com/en/article/what-is-stash-3391708
-# * https://www.deviantart.com/developers/http/v1/20160316/stash_item/4662dd8b10e336486ea9a0b14da62b74
-#
-module Sources
-  module Strategies
-    class Stash < DeviantArt
-      STASH = %r{\Ahttps?://sta\.sh/(?<post_id>[0-9a-zA-Z]+)}i
-
-      def domains
-        ["deviantart.net", "sta.sh"]
-      end
-
-      def match?
-        parsed_urls.map(&:domain).any?("sta.sh")
-      end
-
-      def site_name
-        "Sta.sh"
-      end
-
-      def canonical_url
-        page_url
-      end
-
-      def page_url
-        "https://sta.sh/#{stash_id}"
-      end
-
-      def api_url
-        page_url
-      end
-
-      def self.stash_id_from_url(url)
-       if url =~ STASH
-         $~[:post_id].downcase
-       else
-         nil
-       end
-     end
-
-      def stash_id
-        [url, referer_url].map{ |x| self.class.stash_id_from_url(x) }.compact.first
-      end
-    end
-  end
-end

config/danbooru_default_config.rb

@@ -440,15 +440,6 @@ module Danbooru
       nil
     end
 
-    # Register at https://www.deviantart.com/developers/.
-    def deviantart_client_id
-      nil
-    end
-
-    def deviantart_client_secret
-      nil
-    end
-
     # http://tinysubversions.com/notes/mastodon-bot/
     def pawoo_client_id
       nil

test/unit/artist_test.rb

@@ -230,7 +230,6 @@ class ArtistTest < ActiveSupport::TestCase
 
     context "when finding deviantart artists" do
       setup do
-        skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
         FactoryBot.create(:artist, :name => "artgerm", :url_string => "http://artgerm.deviantart.com/")
         FactoryBot.create(:artist, :name => "trixia",  :url_string => "http://trixdraws.deviantart.com/")
       end

test/unit/downloads/deviant_art_test.rb

@@ -1,19 +0,0 @@
-require 'test_helper'
-
-module Downloads
-  class DeviantArtTest < ActiveSupport::TestCase
-    context "a download for a deviant art html page" do
-      setup do
-        skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
-        
-        @source = "https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484"
-        @download = Downloads::File.new(@source)
-        @tempfile, strategy = @download.download!
-      end
-
-      should "work" do
-        assert_equal(877987, ::File.size(@tempfile.path))
-      end
-    end
-  end
-end

test/unit/sources/deviant_art_test.rb

@@ -2,29 +2,29 @@ require 'test_helper'
 
 module Sources
   class DeviantArtTest < ActiveSupport::TestCase
-    def setup
+    context "A page url" do
-      super
-      skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
-    end
-
-    context "A path-based artist url" do
       setup do
         @site = Sources::Strategies.find("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484")
       end
 
       should "work" do
-        assert_equal(["http://origin-orig.deviantart.net/d533/f/2014/004/8/d/holiday_elincia_by_aeror404-d70rm0s.jpg"], @site.image_urls)
+        # http://origin-orig.deviantart.net/d533/f/2014/004/8/d/holiday_elincia_by_aeror404-d70rm0s.jpg (md5: a7651a6586b95c62fd593dd34bb13618, size: 877987)
-        assert_equal(@site.image_url, @site.canonical_url)
+        assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+        assert_downloaded(877_987, @site.image_url)
+
         assert_equal("aeror404", @site.artist_name)
         assert_equal("https://www.deviantart.com/aeror404", @site.profile_url)
+        assert_equal("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484", @site.page_url)
         assert_equal("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484", @site.page_url_from_image_url)
+        assert_equal("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484", @site.canonical_url)
+        assert_equal("Holiday Elincia", @site.artist_commentary_title)
       end
     end
 
     context "The source for a deleted DeviantArt image URL" do
       should "work" do
         @site = Sources::Strategies.find("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png")
-        @artist = FactoryBot.create(:artist, name: "nickbeja", url_string: "https://nickbeja.deviantart.com")
+        @artist = create(:artist, name: "nickbeja", url_string: "https://nickbeja.deviantart.com")
 
         assert_equal("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png", @site.image_url)
         assert_equal(@site.image_url, @site.canonical_url)
@@ -40,9 +40,17 @@ module Sources
       should "get the image url" do
         @site = Sources::Strategies.find("https://noizave.deviantart.com/art/test-no-download-697415967")
 
-        assert_match(%r!^https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbj81lr-3306feb1-87dc-4d25-9a4c-da8d2973a8b7.jpg!, @site.image_urls[0])
+        # https://img00.deviantart.net/56ee/i/2017/219/2/3/test__no_download_by_noizave-dbj81lr.jpg (md5: 25a03b5a6744b6b914a13b3cd50e3c2c, size: 37638)
-        assert_equal("https://www.deviantart.com/noizave/art/test-no-download-697415967", @site.canonical_url)
+        # orig file: https://danbooru.donmai.us/posts/463438 (md5: eb97244675e47dbd77ffcd2d7e15aeab, size: 59401)
+        assert_match("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbj81lr-3306feb1-87dc-4d25-9a4c-da8d2973a8b7.jpg", @site.image_url)
+        assert_downloaded(40_036, @site.image_url)
+
+        assert_equal("noizave", @site.artist_name)
+        assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
+        assert_equal("test, no download", @site.artist_commentary_title)
+        assert_equal("https://www.deviantart.com/noizave/art/test-no-download-697415967", @site.page_url)
         assert_equal("https://www.deviantart.com/noizave/art/Test-No-Download-697415967", @site.page_url_from_image_url)
+        assert_equal("https://www.deviantart.com/noizave/art/test-no-download-697415967", @site.canonical_url)
       end
     end
 
@@ -50,8 +58,15 @@ module Sources
       should "get the download image url" do
         @site = Sources::Strategies.find("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781")
 
-        assert_equal(["http://origin-orig.deviantart.net/a713/f/2018/333/3/6/all_that_glitters_ii_by_len1-dct67m5.jpg"], @site.image_urls)
+        # http://origin-orig.deviantart.net/a713/f/2018/333/3/6/all_that_glitters_ii_by_len1-dct67m5.jpg (md5: d16bb8620600334caa029ebb9bc426a6, size: 1402017)
-        assert_equal(@site.image_url, @site.canonical_url)
+        assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+        assert_downloaded(1402017, @site.image_url)
+
+        assert_equal("len1", @site.artist_name)
+        assert_equal("https://www.deviantart.com/len1", @site.profile_url)
+        assert_equal("All that Glitters II", @site.artist_commentary_title)
+        assert_equal("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781", @site.page_url)
+        assert_equal("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781", @site.canonical_url)
       end
     end
 
@@ -59,11 +74,16 @@ module Sources
       should "fetch the source data" do
         @site = Sources::Strategies.find("https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg")
 
+        # http://origin-orig.deviantart.net/9e1f/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg (md5: 4cfec3d50ebbb924077cc5c90e705d4e, size: 906621)
+        assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+        assert_downloaded(906_621, @site.image_url)
+
         assert_equal("hideyoshi", @site.artist_name)
         assert_equal("https://www.deviantart.com/hideyoshi", @site.profile_url)
-        assert_equal("http://origin-orig.deviantart.net/9e1f/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg", @site.image_url)
+        assert_equal("https://www.deviantart.com/hideyoshi/art/Legend-of-Galactic-Heroes-635721022", @site.page_url)
         assert_equal("https://www.deviantart.com/hideyoshi/art/Legend-Of-Galactic-Heroes-635721022", @site.page_url_from_image_url)
-        assert_equal(@site.image_url, @site.canonical_url)
+        assert_equal("https://www.deviantart.com/hideyoshi/art/Legend-of-Galactic-Heroes-635721022", @site.canonical_url)
+        assert_equal(%w[barbarossa bay brunhild flare hangar odin planet ship spaceship sun sunset brünhild legendsofgalacticheroes].sort, @site.tags.map(&:first).sort)
       end
     end
 
@@ -71,10 +91,14 @@ module Sources
       should "work" do
         @site = Sources::Strategies.find("http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png")
 
-        assert_equal(@site.url, @site.image_url)
+        # md5: 9dec050536dbdb09ab63cb9c5a48f8b7
+        assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+        assert_downloaded(3619, @site.image_url)
+
         assert_equal("https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", @site.page_url)
         assert_equal("https://www.deviantart.com/noizave/art/Test-Post-Please-Ignore-685436408", @site.page_url_from_image_url)
-        assert_equal(@site.image_url, @site.canonical_url)
+        assert_equal("https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", @site.canonical_url)
+
         assert_equal("noizave", @site.artist_name)
         assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
         assert_equal(%w[bar baz foo], @site.tags.map(&:first))
@@ -85,7 +109,10 @@ module Sources
     context "The source for a img00.deviantart.net sample image url" do
       should "return the full size image url" do
         @site = Sources::Strategies.find("https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png")
-        assert_equal("http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png", @site.image_url)
+
+        assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+        assert_downloaded(3619, @site.image_url)
+
         assert_equal("https://www.deviantart.com/noizave/art/Test-Post-Please-Ignore-685436408", @site.page_url_from_image_url)
       end
     end
@@ -93,7 +120,11 @@ module Sources
     context "The source for a th00.deviantart.net/*/PRE/* thumbnail url" do
       should "return the full size image url" do
         @site = Sources::Strategies.find("http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png")
-        assert_equal("http://origin-orig.deviantart.net/0f1e/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png", @site.image_url)
+
+        # http://origin-orig.deviantart.net/0f1e/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png (md5: d779f5a7da29ec90d777a8db38d07994, size: 3391584)
+        assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+        assert_downloaded(3_391_584, @site.image_url)
+
         assert_equal("https://www.deviantart.com/xyelkiltrox/art/Goruto-438744629", @site.page_url_from_image_url)
       end
     end
@@ -101,7 +132,7 @@ module Sources
     context "A source for a *.deviantart.net/*/:title_by_:artist.jpg url artist name containing underscores" do
       should "find the correct artist" do
         @site = Sources::Strategies.find("https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg")
-        @artist = FactoryBot.create(:artist, name: "mikoto-chan", url_string: "https://www.deviantart.com/mikoto-chan")
+        @artist = create(:artist, name: "mikoto-chan", url_string: "https://www.deviantart.com/mikoto-chan")
 
         assert_equal("mikoto-chan", @site.artist_name)
         assert_equal([@artist], @site.artists)
@@ -113,7 +144,7 @@ module Sources
       setup do
         @url = "http://fc08.deviantart.net/files/f/2007/120/c/9/cool_like_me_by_47ness.jpg"
         @ref = "https://47ness.deviantart.com/art/Cool-Like-Me-54339311"
-        @artist = FactoryBot.create(:artist, name: "47ness", url_string: "https://www.deviantart.com/47ness")
+        @artist = create(:artist, name: "47ness", url_string: "https://www.deviantart.com/47ness")
       end
 
       context "without a referer" do
@@ -123,7 +154,7 @@ module Sources
           assert_equal(@site.url, @site.image_url)
           assert_equal("47ness", @site.artist_name)
           assert_equal("https://www.deviantart.com/47ness", @site.profile_url)
-          assert_equal("", @site.page_url)
+          assert_nil(@site.page_url)
           assert_nil(@site.page_url_from_image_url)
           assert_equal(@site.image_url, @site.canonical_url)
           assert_equal([@artist], @site.artists)
@@ -135,7 +166,10 @@ module Sources
         should "work" do
           @site = Sources::Strategies.find(@url, @ref)
 
-          assert_equal("http://origin-orig.deviantart.net/a418/f/2007/120/c/9/cool_like_me_by_47ness.jpg", @site.image_url)
+          # http://origin-orig.deviantart.net/a418/f/2007/120/c/9/cool_like_me_by_47ness.jpg (md5: da78e7c192d42470acda7d87ade64849, size: 265496)
+          assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+          assert_downloaded(265_496, @site.image_url)
+
           assert_equal("47ness", @site.artist_name)
           assert_equal("https://www.deviantart.com/47ness", @site.profile_url)
           assert_equal("https://www.deviantart.com/47ness/art/Cool-Like-Me-54339311", @site.page_url)
@@ -150,7 +184,7 @@ module Sources
       setup do
         @url = "http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg"
         @ref = "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896"
-        @artist = FactoryBot.create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com")
+        @artist = create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com")
       end
 
       context "without a referer" do
@@ -158,9 +192,9 @@ module Sources
           @site = Sources::Strategies.find(@url)
 
           assert_equal(@url, @site.image_url)
-          assert_equal("", @site.artist_name)
+          assert_nil(@site.artist_name)
-          assert_equal("", @site.profile_url)
+          assert_nil(@site.profile_url)
-          assert_equal("", @site.page_url)
+          assert_nil(@site.page_url)
           assert_nil(@site.page_url_from_image_url)
           assert_equal(@site.image_url, @site.canonical_url)
           assert_equal([], @site.artists)
@@ -172,7 +206,7 @@ module Sources
         should "work" do
           @site = Sources::Strategies.find(@url, @ref)
 
-          assert_match(%r!^https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_url)
+          assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_url)
           assert_equal("edsfox", @site.artist_name)
           assert_equal("https://www.deviantart.com/edsfox", @site.profile_url)
           assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", @site.page_url)
@@ -187,14 +221,14 @@ module Sources
       setup do
         @url = "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg"
         @ref = "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896"
-        @artist = FactoryBot.create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com")
+        @artist = create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com")
       end
 
       context "with a referer" do
         should "work" do
           @site = Sources::Strategies.find(@url, @ref)
 
-          assert_match(%r!^https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_url)
+          assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_url)
           assert_equal("edsfox", @site.artist_name)
           assert_equal("https://www.deviantart.com/edsfox", @site.profile_url)
           assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", @site.page_url)
@@ -211,7 +245,9 @@ module Sources
       end
 
       should "get the image url" do
-        assert_match(%r!https?://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png!, @site.image_url)
+        # https://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png
+        assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+        assert_downloaded(3619, @site.image_url)
       end
 
       should "get the profile" do
@@ -269,7 +305,9 @@ module Sources
       end
 
       should "get the image url" do
-        assert_match(%r!https?://origin-orig\.deviantart\.net/cb25/f/2017/160/1/9/hidden_work_by_noizave-dbc3r29\.png!, @site.image_url)
+        # https://origin-orig.deviantart.net/cb25/f/2017/160/1/9/hidden_work_by_noizave-dbc3r29.png
+        assert_match(%r!\Ahttps://api-da.wixmp.com/_api/download/file\?downloadToken=!, @site.image_url)
+        assert_downloaded(3619, @site.image_url)
       end
     end
 
@@ -284,7 +322,7 @@ module Sources
     context "An artist entry with a profile url that is missing the 'www'" do
       should "still find the artist" do
         @site = Sources::Strategies.find("http://noizave.deviantart.com/art/test-post-please-ignore-685436408")
-        @artist = FactoryBot.create(:artist, name: "noizave", url_string: "https://deviantart.com/noizave")
+        @artist = create(:artist, name: "noizave", url_string: "https://deviantart.com/noizave")
 
         assert_equal([@artist], @site.artists)
       end

test/unit/sources/stash_test.rb

@@ -1,58 +0,0 @@
-require 'test_helper'
-
-module Sources
-  class StashTest < ActiveSupport::TestCase
-    def setup
-      super
-      skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
-    end
-
-    context "A https://sta.sh/:id page url" do
-      should "work" do
-        @site = Sources::Strategies.find("https://sta.sh/0wxs31o7nn2")
-
-        assert_equal("noizave", @site.artist_name)
-        assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
-
-        assert_equal("A pepe", @site.artist_commentary_title)
-        assert_equal("This is a test.", @site.artist_commentary_desc)
-
-        assert_equal("https://sta.sh/0wxs31o7nn2", @site.page_url)
-        assert_equal("https://sta.sh/0wxs31o7nn2", @site.canonical_url)
-        assert_equal("http://origin-orig.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png", @site.image_url)
-        assert_equal(["http://origin-orig.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png"], @site.image_urls)
-      end
-    end
-
-    context "A https://orig00.deviantart.net/* image url" do
-      context "with a https://sta.sh/:id referer" do
-        should "work" do
-          @site = Sources::Strategies.find("https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png", "https://sta.sh/0wxs31o7nn2")
-
-          assert_equal("noizave", @site.artist_name)
-          assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
-
-          assert_equal("A pepe", @site.artist_commentary_title)
-          assert_equal("This is a test.", @site.artist_commentary_desc)
-
-          assert_equal("https://sta.sh/0wxs31o7nn2", @site.page_url)
-          assert_equal("https://sta.sh/0wxs31o7nn2", @site.canonical_url)
-          assert_equal("http://origin-orig.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png", @site.image_url)
-          assert_equal(["http://origin-orig.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png"], @site.image_urls)
-        end
-      end
-
-      context "without a referer" do
-        should "use the base deviantart strategy" do
-          @site = Sources::Strategies.find("https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png")
-
-          # if all we have is the image url, then we can't tell that this is really a sta.sh image.
-          assert_equal("Deviant Art", @site.site_name)
-
-          # this is the wrong page, but there's no way to know the correct sta.sh page without the referer.
-          assert_equal("https://www.deviantart.com/deviation/763305148", @site.page_url)
-        end
-      end
-    end
-  end
-end