jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #5188 from nonamethanks/fix-deviantart

Browse Source 
Fix deviantart strategy to get biggest available size
This commit is contained in:
evazion
2022-06-01 18:39:43 -05:00
committed by GitHub
parent 434e44ba8b 3a0a32b98a
commit cc2285ed6f
3 changed files with 198 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
app/logical/source/extractor/deviant_art.rb
View File

@@ -18,7 +18,11 @@ module Source
def image_url def image_url
# work is private, deleted, or the url didn't contain a deviation id; use image url as given by user. # work is private, deleted, or the url didn't contain a deviation id; use image url as given by user.
if api_deviation.blank? if api_deviation.blank?
if url =~ %r{\Ahttps://images-wixmp-}
extract_largest(url)
else
url url
end
elsif api_deviation[:is_downloadable] elsif api_deviation[:is_downloadable]
api_download[:src] api_download[:src]
elsif api_deviation[:flash].present? elsif api_deviation[:flash].present?
@@ -27,13 +31,28 @@ module Source
api_deviation[:videos].max_by { |x| x[:filesize] }[:src] api_deviation[:videos].max_by { |x| x[:filesize] }[:src]
else else
src = api_deviation.dig(:content, :src) src = api_deviation.dig(:content, :src)
if deviation_id && deviation_id.to_i <= 790_677_560 && src =~ %r{\Ahttps://images-wixmp-} && src !~ /\.gif\?/ extract_largest(src)
src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1')
src = src.sub(%r{/v1/(fit|fill)/.*\z}i, "")
end end
src = src.sub(%r{\Ahttps?://orig\d+\.deviantart\.net}i, "http://origin-orig.deviantart.net") end
src = src.gsub(/q_\d+,strp/, "q_100")
def extract_largest(src)
if src =~ %r{\Ahttps://images-wixmp-}
sample, separator, * = src.partition("/v1/")
if separator.blank?
src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1') unless src =~ /\.gif\?/
src src
else
# :^) https://i.imgur.com/KG5bVRU.png
# shamelessly aped from:
# https://github.com/mikf/gallery-dl/blob/7990fe84f11271bc8e4079db6b0248dbeb79474a/gallery_dl/extractor/deviantart.py#L293
*, f_value = sample.split("/f/")
data = {sub: "urn:app:", iss: "urn:app:", obj: [[{path: "/f/#{f_value}"}]], aud: ["urn:service:file.download"]}
token = Base64.encode64(JSON.generate(data)).gsub("=", "").gsub("\n", "")
"#{sample}?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.#{token}."
end
else
src = src.sub(%r{\Ahttps?://orig\d+\.deviantart\.net}i, "http://origin-orig.deviantart.net")
src.gsub(/q_\d+,strp/, "q_100")
end end
end end

17
test/test_helpers/source_test_helper.rb
View File

@@ -86,15 +86,22 @@ module SourceTestHelper
# check any method that is passed as kwargs, in order to hardcode as few things as possible # check any method that is passed as kwargs, in order to hardcode as few things as possible
# XXX can't use **kwargs because of a bug with shoulda-context, so we're using a hash temporarily # XXX can't use **kwargs because of a bug with shoulda-context, so we're using a hash temporarily
methods_to_test.each do |method_name, expected_value| methods_to_test.each do |method_name, expected_value|
actual_value = strategy.try(method_name)
should "make sure that '#{method_name}' matches" do should "make sure that '#{method_name}' matches" do
if expected_value.instance_of? Regexp if expected_value.instance_of? Regexp
assert_match(expected_value, strategy.try(method_name)) assert_match(expected_value, actual_value)
elsif expected_value.instance_of? Array elsif expected_value.instance_of? Array
assert_equal(expected_value.sort, strategy.try(method_name).sort) if expected_value.first.instance_of? Regexp
elsif expected_value.nil? actual_values = actual_value.sort
assert_nil(strategy.try(method_name)) expected_value.sort.each_with_index { |each_value, index| assert_match(each_value, actual_values[index]) }
else else
assert_equal(expected_value, strategy.try(method_name)) assert_equal(expected_value.sort, actual_value)
end
elsif expected_value.nil?
assert_nil(actual_value)
else
assert_equal(expected_value, actual_value)
end end
end end
end end

468
test/unit/sources/deviant_art_test.rb
View File

@@ -1,4 +1,4 @@
require 'test_helper' require "test_helper"
module Sources module Sources
class DeviantArtTest < ActiveSupport::TestCase class DeviantArtTest < ActiveSupport::TestCase
@@ -7,307 +7,42 @@ module Sources
skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present? skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
end end
context "A page url" do context "A deviantart post" do
setup do strategy_should_work(
@site = Source::Extractor.find("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484") "https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/11a24395-0f24-446d-ae73-a9f812e79e55/d70rm0s-e5b6b5e6-5795-44bb-a0ba-27b5c2349be7\.jpg}],
download_size: 877_987,
page_url: "https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484",
artist_name: "aeror404",
profile_url: "https://www.deviantart.com/aeror404",
artist_commentary_title: "Holiday Elincia"
)
end end
should "work" do context "A deviantart image" do
# http://origin-orig.deviantart.net/d533/f/2014/004/8/d/holiday_elincia_by_aeror404-d70rm0s.jpg (md5: a7651a6586b95c62fd593dd34bb13618, size: 877987) strategy_should_work(
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/11a24395-0f24-446d-ae73-a9f812e79e55/d70rm0s-e5b6b5e6-5795-44bb-a0ba-27b5c2349be7\.jpg}, @site.image_urls.sole) "https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png",
assert_downloaded(877_987, @site.image_urls.sole) image_urls: ["https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png"],
download_size: 840_296,
assert_equal("aeror404", @site.artist_name) page_url: "https://www.deviantart.com/nickbeja/art/Mindflayer-Girl01-708675884",
assert_equal("https://www.deviantart.com/aeror404", @site.profile_url) artist_name: "nickbeja",
assert_equal("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484", @site.page_url) profile_url: "https://www.deviantart.com/nickbeja"
assert_equal("Holiday Elincia", @site.artist_commentary_title) )
end
end end
context "The source for a deleted DeviantArt image URL" do context "Another deviantart image" do
should "work" do strategy_should_work(
@site = Source::Extractor.find("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png") "https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg",
@artist = create(:artist, name: "nickbeja", url_string: "https://nickbeja.deviantart.com") image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/b1f96af6-56a3-47a8-b7f4-406f243af3a3/daihpha-9f1fcd2e-7557-4db5-951b-9aedca9a3ae7\.jpg}],
download_size: 906_621,
assert_equal(["https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png"], @site.image_urls) page_url: "https://www.deviantart.com/hideyoshi/art/Legend-of-Galactic-Heroes-635721022",
assert_equal("nickbeja", @site.artist_name) artist_name: "hideyoshi",
assert_equal("https://www.deviantart.com/nickbeja", @site.profile_url) profile_url: "https://www.deviantart.com/hideyoshi",
assert_equal("https://www.deviantart.com/nickbeja/art/Mindflayer-Girl01-708675884", @site.page_url) tags: %w[barbarossa bay brunhild flare hangar odin planet ship spaceship sun sunset brünhild legendsofgalacticheroes]
assert_equal([@artist], @site.artists) )
assert_nothing_raised { @site.to_h }
end
end end
context "The source for a download-disabled DeviantArt artwork page" do context "A deviantart origin-orig image" do
should "get the image url" do
@site = Source::Extractor.find("https://noizave.deviantart.com/art/test-no-download-697415967")
# https://img00.deviantart.net/56ee/i/2017/219/2/3/test__no_download_by_noizave-dbj81lr.jpg (md5: 25a03b5a6744b6b914a13b3cd50e3c2c, size: 37638)
# orig file: https://danbooru.donmai.us/posts/463438 (md5: eb97244675e47dbd77ffcd2d7e15aeab, size: 59401)
assert_match("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbj81lr-3306feb1-87dc-4d25-9a4c-da8d2973a8b7.jpg", @site.image_urls.sole)
assert_downloaded(40_036, @site.image_urls.sole)
assert_equal("noizave", @site.artist_name)
assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
assert_equal("test, no download", @site.artist_commentary_title)
assert_equal("https://www.deviantart.com/noizave/art/test-no-download-697415967", @site.page_url)
end
end
context "The source for a download-enabled DeviantArt artwork page" do
should "get the download image url" do
@site = Source::Extractor.find("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781")
# http://origin-orig.deviantart.net/a713/f/2018/333/3/6/all_that_glitters_ii_by_len1-dct67m5.jpg (md5: d16bb8620600334caa029ebb9bc426a6, size: 1402017)
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/a6289ca5-2205-4118-af55-c6934fba0930/dct67m5-51e8db38-9167-4f5c-931d-561ea4d3810d\.jpg}, @site.image_urls.sole)
assert_downloaded(1402017, @site.image_urls.sole)
assert_equal("len1", @site.artist_name)
assert_equal("https://www.deviantart.com/len1", @site.profile_url)
assert_equal("All that Glitters II", @site.artist_commentary_title)
assert_equal("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781", @site.page_url)
end
end
context "The source for a DeviantArt image url" do
should "fetch the source data" do
@site = Source::Extractor.find("https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg")
# http://origin-orig.deviantart.net/9e1f/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg (md5: 4cfec3d50ebbb924077cc5c90e705d4e, size: 906621)
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/b1f96af6-56a3-47a8-b7f4-406f243af3a3/daihpha-9f1fcd2e-7557-4db5-951b-9aedca9a3ae7\.jpg}, @site.image_urls.sole)
assert_downloaded(906_621, @site.image_urls.sole)
assert_equal("hideyoshi", @site.artist_name)
assert_equal("https://www.deviantart.com/hideyoshi", @site.profile_url)
assert_equal("https://www.deviantart.com/hideyoshi/art/Legend-of-Galactic-Heroes-635721022", @site.page_url)
assert_equal(%w[barbarossa bay brunhild flare hangar odin planet ship spaceship sun sunset brünhild legendsofgalacticheroes].sort, @site.tags.map(&:first).sort)
end
end
context "The source for a origin-orig.deviantart.net image url without a referer" do
should "work" do
@site = Source::Extractor.find("http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png")
# md5: 9dec050536dbdb09ab63cb9c5a48f8b7
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole)
assert_downloaded(3619, @site.image_urls.sole)
assert_equal("https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", @site.page_url)
assert_equal("noizave", @site.artist_name)
assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
assert_equal(%w[bar baz foo], @site.tags.map(&:first))
assert_nothing_raised { @site.to_h }
end
end
context "The source for a img00.deviantart.net sample image url" do
should "return the full size image url" do
@site = Source::Extractor.find("https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png")
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole)
assert_downloaded(3619, @site.image_urls.sole)
assert_equal("https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", @site.page_url)
end
end
context "The source for a th00.deviantart.net/*/PRE/* thumbnail url" do
should "return the full size image url" do
@site = Source::Extractor.find("http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png")
# http://origin-orig.deviantart.net/0f1e/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png (md5: d779f5a7da29ec90d777a8db38d07994, size: 3391584)
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/d8995973-0b32-4a7d-8cd8-d847d083689a/d797tit-1eac22e0-38b6-4eae-adcb-1b72843fd62a\.png}, @site.image_urls.sole)
assert_downloaded(3_391_584, @site.image_urls.sole)
assert_equal("https://www.deviantart.com/xyelkiltrox/art/Goruto-438744629", @site.page_url)
end
end
context "A source for a *.deviantart.net/*/:title_by_:artist.jpg url artist name containing underscores" do
should "find the correct artist" do
@site = Source::Extractor.find("https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg")
@artist = create(:artist, name: "mikoto-chan", url_string: "https://www.deviantart.com/mikoto-chan")
assert_equal("mikoto-chan", @site.artist_name)
assert_equal([@artist], @site.artists)
assert_nil(@site.page_url)
end
end
context "The source for a *.deviantart.net/*/:title_by_:artist.jpg url" do
setup do
@url = "http://fc08.deviantart.net/files/f/2007/120/c/9/cool_like_me_by_47ness.jpg"
@ref = "https://47ness.deviantart.com/art/Cool-Like-Me-54339311"
@artist = create(:artist, name: "47ness", url_string: "https://www.deviantart.com/47ness")
end
context "without a referer" do
should "work" do
@site = Source::Extractor.find(@url)
assert_equal([@site.url], @site.image_urls)
assert_equal("47ness", @site.artist_name)
assert_equal("https://www.deviantart.com/47ness", @site.profile_url)
assert_nil(@site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
context "with a referer" do
should "work" do
@site = Source::Extractor.find(@url, @ref)
# http://origin-orig.deviantart.net/a418/f/2007/120/c/9/cool_like_me_by_47ness.jpg (md5: da78e7c192d42470acda7d87ade64849, size: 265496)
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ece2238f-5c8f-48e4-afda-304cab294acd/dwcohb-8189be91-691d-4212-b3a0-0b77e86a57d1\.jpg}, @site.image_urls.sole)
assert_downloaded(265_496, @site.image_urls.sole)
assert_equal("47ness", @site.artist_name)
assert_equal("https://www.deviantart.com/47ness", @site.profile_url)
assert_equal("https://www.deviantart.com/47ness/art/Cool-Like-Me-54339311", @site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
end
context "The source for a *.deviantart.net/*/:hash.jpg url" do
setup do
@url = "http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg"
@ref = "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896"
@artist = create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com")
end
context "without a referer" do
should "work" do
@site = Source::Extractor.find(@url)
assert_equal([@url], @site.image_urls)
assert_nil(@site.artist_name)
assert_nil(@site.profile_url)
assert_nil(@site.page_url)
assert_equal([], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
context "with a referer" do
should "work" do
@site = Source::Extractor.find(@url, @ref)
assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_urls.sole)
assert_equal("edsfox", @site.artist_name)
assert_equal("https://www.deviantart.com/edsfox", @site.profile_url)
assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", @site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
end
context "The source for a images-wixmp-.* sample image" do
setup do
@url = "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg"
@ref = "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896"
@artist = create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com")
end
context "with a referer" do
should "work" do
@site = Source::Extractor.find(@url, @ref)
assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_urls.sole)
assert_equal("edsfox", @site.artist_name)
assert_equal("https://www.deviantart.com/edsfox", @site.profile_url)
assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", @site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
end
context "The source for a api-da.wixmp.com image" do
setup do
@url = "https://api-da.wixmp.com/_api/download/file?downloadToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsImV4cCI6MTU5MDkwMTUzMywiaWF0IjoxNTkwOTAwOTIzLCJqdGkiOiI1ZWQzMzhjNWQ5YjI0Iiwib2JqIjpudWxsLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdLCJwYXlsb2FkIjp7InBhdGgiOiJcL2ZcL2U0NmE0OGViLTNkMGItNDQ5ZS05MGRjLTBhMWIzMWNiMTM2MVwvZGQzcDF4OS1mYjQ3YmM4Zi02NTNlLTQyYTItYmI0ZC1hZmFmOWZjMmI3ODEuanBnIn19.-zo8E2eDmkmDNCK-sMabBajkaGtVYJ2Q20iVrUtt05Q"
@ref = "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029"
@artist = create(:artist, name: "akizero", url_string: "https://akizero1510.deviantart.com")
end
context "with a referer" do
should "work" do
@site = Source::Extractor.find(@url, @ref)
assert_equal(@ref, @site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
end
context "The source for a non-downloadable animated gif with id<=790677560" do
should "return working image url" do
@site = Source::Extractor.find("https://www.deviantart.com/heartgear/art/Silent-Night-579982816")
# md5: 62caac1863aa264a56d548b4b7607097
assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ea95be00-c5aa-4063-bd55-f5a9183912f7/d9lb1ls-7d625444-0003-4123-bf00-274737ca7fdd.gif\?token=!, @site.image_urls.sole)
assert_downloaded(350_156, @site.image_urls.sole)
end
end
context "The source for a non-downloadable flash file" do
should "return working image url" do
skip
@site = Source::Extractor.find("https://www.deviantart.com/heartgear/art/SL-40v3-522007633")
# md5: 6adf1a3d532f898f44cf9948cbc7db7d
assert_match(%r!\Ahttps://api-da\.wixmp\.com/_api/download/file\?downloadToken=!, @site.image_urls.sole)
assert_downloaded(3_496_110, @site.image_urls.sole)
end
end
context "The source for a non-downloadable video file" do
should "return working image url" do
@site = Source::Extractor.find("https://www.deviantart.com/gs-mantis/art/Chen-Goes-Fishing-505847233")
# md5: 344ac2b9fd5a87982af4b648aa2b2b0d
assert_equal(["https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/v/mp4/fe046bc7-4d68-4699-96c1-19aa464edff6/d8d6281-91959e92-214f-4b2d-a138-ace09f4b6d09.1080p.8e57939eba634743a9fa41185e398d00.mp4"], @site.image_urls)
assert_downloaded(9_739_947, @site.image_urls.sole)
end
end
context "The source for an DeviantArt artwork page" do
setup do
@site = Source::Extractor.find("http://noizave.deviantart.com/art/test-post-please-ignore-685436408")
end
should "get the image url" do
# https://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole)
assert_downloaded(3619, @site.image_urls.sole)
end
should "get the profile" do
assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
end
should "get the artist name" do
assert_equal("noizave", @site.artist_name)
end
should "get the tags" do
assert_equal(%w[bar baz foo], @site.tags.map(&:first))
end
should "get the artist commentary" do
title = "test post please ignore"
desc = "<div align=\"center\"><span>blah blah<br /><div align=\"left\"><a class=\"external\" href=\"https://www.deviantart.com/users/outgoing?http://www.google.com\">test link</a><br /></div></span></div><br /><h1>lol</h1><br /><br /><b>blah</b>&nbsp;<i>blah</i>&nbsp;<u>blah</u>&nbsp;<strike>blah</strike><br />herp derp<br /><br /><blockquote>this is a quote</blockquote><ol><li>one</li><li>two</li><li>three</li></ol><ul><li>one</li><li>two</li><li>three</li></ul><img src=\"https://e.deviantart.net/emoticons/h/heart.gif\" alt=\"Heart\" style=\"width: 15px; height: 13px;\" data-embed-type=\"emoticon\" data-embed-id=\"357\">&nbsp;&nbsp;"
assert_equal(title, @site.artist_commentary_title)
assert_equal(desc, @site.artist_commentary_desc)
end
should "get the dtext-ified commentary" do
desc = <<-EOS.strip_heredoc.chomp desc = <<-EOS.strip_heredoc.chomp
blah blah blah blah
"test link":[http://www.google.com] "test link":[http://www.google.com]
@@ -332,27 +67,146 @@ module Sources
"Heart":[https://e.deviantart.net/emoticons/h/heart.gif] "Heart":[https://e.deviantart.net/emoticons/h/heart.gif]
EOS EOS
assert_equal(desc, @site.dtext_artist_commentary_desc) strategy_should_work(
end "http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}],
download_size: 3_619,
page_url: "https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408",
artist_name: "noizave",
profile_url: "https://www.deviantart.com/noizave",
tags: %w[bar baz foo],
artist_commentary_title: "test post please ignore",
dtext_artist_commentary_desc: desc
)
end end
context "The source for a login-only DeviantArt artwork page" do context "A img00.deviantart.net sample" do
setup do strategy_should_work(
@site = Source::Extractor.find("http://noizave.deviantart.com/art/hidden-work-685458369") "https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}],
download_size: 3_619,
page_url: "https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408"
)
end end
should "get the image url" do context "A th00.deviantart.net/*/PRE/* thumbnail" do
# https://origin-orig.deviantart.net/cb25/f/2017/160/1/9/hidden_work_by_noizave-dbc3r29.png strategy_should_work(
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3r29-10c99118-5cfe-4402-ad55-7b57e7c0ca43\.png}, @site.image_urls.sole) "http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png",
assert_downloaded(3619, @site.image_urls.sole) image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/d8995973-0b32-4a7d-8cd8-d847d083689a/d797tit-1eac22e0-38b6-4eae-adcb-1b72843fd62a\.png}],
download_size: 3_391_584,
page_url: "https://www.deviantart.com/xyelkiltrox/art/Goruto-438744629"
)
end end
context "A deviantart page with download disabled" do
strategy_should_work(
"https://noizave.deviantart.com/art/test-no-download-697415967",
image_urls: [%r{https://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbj81lr-3306feb1-87dc-4d25-9a4c-da8d2973a8b7\.jpg\?token=}],
download_size: 59_401,
page_url: "https://www.deviantart.com/noizave/art/test-no-download-697415967",
artist_name: "noizave",
profile_url: "https://www.deviantart.com/noizave",
artist_commentary_title: "test, no download"
)
end
context "A deviantart page with download disabled for a huge file" do
strategy_should_work(
"https://www.deviantart.com/anatofinnstark/art/The-Blade-of-Miquella-914166242",
download_size: 26_037_561
)
end
context "A deviantart page with download enabled" do
strategy_should_work(
"https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/a6289ca5-2205-4118-af55-c6934fba0930/dct67m5-51e8db38-9167-4f5c-931d-561ea4d3810d\.jpg}],
download_size: 1_402_017,
page_url: "https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781",
artist_name: "len1",
profile_url: "https://www.deviantart.com/len1",
artist_commentary_title: "All that Glitters II"
)
end
context "A *.deviantart.net/*/:title_by_:artist.jpg image with an artist name containing underscores" do
strategy_should_work(
"https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg",
image_urls: ["https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg"],
artist_name: "mikoto-chan",
profile_url: "https://www.deviantart.com/mikoto-chan",
page_url: nil
)
end
context "A *.deviantart.net/*/:hash.jpg image without referer" do
strategy_should_work(
"http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg",
image_urls: ["http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg"],
profile_url: nil,
page_url: nil
)
end
context "A *.deviantart.net/*/:hash.jpg image with referer" do
strategy_should_work(
"http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg",
referer: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896",
image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg}],
artist_name: "edsfox",
profile_url: "https://www.deviantart.com/edsfox",
page_url: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896"
)
end
context "A images-wixmp-.* sample" do
strategy_should_work(
"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg",
referer: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896",
image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg}],
artist_name: "edsfox",
profile_url: "https://www.deviantart.com/edsfox",
page_url: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896"
)
end
context "A api-da.wixmp.com sample" do
strategy_should_work(
"https://api-da.wixmp.com/_api/download/file?downloadToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsImV4cCI6MTU5MDkwMTUzMywiaWF0IjoxNTkwOTAwOTIzLCJqdGkiOiI1ZWQzMzhjNWQ5YjI0Iiwib2JqIjpudWxsLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdLCJwYXlsb2FkIjp7InBhdGgiOiJcL2ZcL2U0NmE0OGViLTNkMGItNDQ5ZS05MGRjLTBhMWIzMWNiMTM2MVwvZGQzcDF4OS1mYjQ3YmM4Zi02NTNlLTQyYTItYmI0ZC1hZmFmOWZjMmI3ODEuanBnIn19.-zo8E2eDmkmDNCK-sMabBajkaGtVYJ2Q20iVrUtt05Q",
referer: "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029",
page_url: "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029"
)
end
context "A non-downloadable animated gif with id<=790677560" do
strategy_should_work(
"https://www.deviantart.com/heartgear/art/Silent-Night-579982816",
image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ea95be00-c5aa-4063-bd55-f5a9183912f7/d9lb1ls-7d625444-0003-4123-bf00-274737ca7fdd.gif\?token=}],
download_size: 350_156
)
end
context "A non-downloadable video file" do
strategy_should_work(
"https://www.deviantart.com/gs-mantis/art/Chen-Goes-Fishing-505847233",
image_urls: ["https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/v/mp4/fe046bc7-4d68-4699-96c1-19aa464edff6/d8d6281-91959e92-214f-4b2d-a138-ace09f4b6d09.1080p.8e57939eba634743a9fa41185e398d00.mp4"],
download_size: 9_739_947
)
end
context "A login-only deviantart post" do
strategy_should_work(
"http://noizave.deviantart.com/art/hidden-work-685458369",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3r29-10c99118-5cfe-4402-ad55-7b57e7c0ca43\.png}],
download_size: 3_619
)
end end
context "A source with malformed links in the artist commentary" do context "A source with malformed links in the artist commentary" do
should "fix the links" do should "fix the links" do
@site = Source::Extractor.find("https://teemutaiga.deviantart.com/art/Kisu-620666655") @site = Source::Extractor.find("https://teemutaiga.deviantart.com/art/Kisu-620666655")
assert_match(%r!"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]!, @site.dtext_artist_commentary_desc) assert_match(%r{"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]}, @site.dtext_artist_commentary_desc)
end end
end end
@@ -393,7 +247,7 @@ module Sources
assert(Source::URL.page_url?("https://sta.sh/0wxs31o7nn2")) assert(Source::URL.page_url?("https://sta.sh/0wxs31o7nn2"))
assert(Source::URL.profile_url?("https://www.deviantart.com/noizave")) assert(Source::URL.profile_url?("https://www.deviantart.com/noizave"))
assert(Source::URL.profile_url?("https://noizave.deviantart.com")) assert(Source::URL.profile_url?("https://noizave.deviantart.com"))
refute(Source::URL.profile_url?("https://deviantart.net")) assert_not(Source::URL.profile_url?("https://deviantart.net"))
end end
end end
end end