jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #5188 from nonamethanks/fix-deviantart

Browse Source 
Fix deviantart strategy to get biggest available size
This commit is contained in:
evazion
2022-06-01 18:39:43 -05:00
committed by GitHub
parent 434e44ba8b 3a0a32b98a
commit cc2285ed6f
3 changed files with 198 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
app/logical/source/extractor/deviant_art.rb
View File

@@ -18,7 +18,11 @@ module Source
def image_url def image_url
# work is private, deleted, or the url didn't contain a deviation id; use image url as given by user. # work is private, deleted, or the url didn't contain a deviation id; use image url as given by user.
if api_deviation.blank? if api_deviation.blank?
url if url =~ %r{\Ahttps://images-wixmp-}
extract_largest(url)
else
url
end
elsif api_deviation[:is_downloadable] elsif api_deviation[:is_downloadable]
api_download[:src] api_download[:src]
elsif api_deviation[:flash].present? elsif api_deviation[:flash].present?
@@ -27,13 +31,28 @@ module Source
api_deviation[:videos].max_by { |x| x[:filesize] }[:src] api_deviation[:videos].max_by { |x| x[:filesize] }[:src]
else else
src = api_deviation.dig(:content, :src) src = api_deviation.dig(:content, :src)
if deviation_id && deviation_id.to_i <= 790_677_560 && src =~ %r{\Ahttps://images-wixmp-} && src !~ /\.gif\?/ extract_largest(src)
src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1') end
src = src.sub(%r{/v1/(fit|fill)/.*\z}i, "") end
def extract_largest(src)
if src =~ %r{\Ahttps://images-wixmp-}
sample, separator, * = src.partition("/v1/")
if separator.blank?
src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1') unless src =~ /\.gif\?/
src
else
# :^) https://i.imgur.com/KG5bVRU.png
# shamelessly aped from:
# https://github.com/mikf/gallery-dl/blob/7990fe84f11271bc8e4079db6b0248dbeb79474a/gallery_dl/extractor/deviantart.py#L293
*, f_value = sample.split("/f/")
data = {sub: "urn:app:", iss: "urn:app:", obj: [[{path: "/f/#{f_value}"}]], aud: ["urn:service:file.download"]}
token = Base64.encode64(JSON.generate(data)).gsub("=", "").gsub("\n", "")
"#{sample}?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.#{token}."
end end
else
src = src.sub(%r{\Ahttps?://orig\d+\.deviantart\.net}i, "http://origin-orig.deviantart.net") src = src.sub(%r{\Ahttps?://orig\d+\.deviantart\.net}i, "http://origin-orig.deviantart.net")
src = src.gsub(/q_\d+,strp/, "q_100") src.gsub(/q_\d+,strp/, "q_100")
src
end end
end end

15
test/test_helpers/source_test_helper.rb
View File

@@ -86,15 +86,22 @@ module SourceTestHelper
# check any method that is passed as kwargs, in order to hardcode as few things as possible # check any method that is passed as kwargs, in order to hardcode as few things as possible
# XXX can't use **kwargs because of a bug with shoulda-context, so we're using a hash temporarily # XXX can't use **kwargs because of a bug with shoulda-context, so we're using a hash temporarily
methods_to_test.each do |method_name, expected_value| methods_to_test.each do |method_name, expected_value|
actual_value = strategy.try(method_name)
should "make sure that '#{method_name}' matches" do should "make sure that '#{method_name}' matches" do
if expected_value.instance_of? Regexp if expected_value.instance_of? Regexp
assert_match(expected_value, strategy.try(method_name)) assert_match(expected_value, actual_value)
elsif expected_value.instance_of? Array elsif expected_value.instance_of? Array
assert_equal(expected_value.sort, strategy.try(method_name).sort) if expected_value.first.instance_of? Regexp
actual_values = actual_value.sort
expected_value.sort.each_with_index { |each_value, index| assert_match(each_value, actual_values[index]) }
else
assert_equal(expected_value.sort, actual_value)
end
elsif expected_value.nil? elsif expected_value.nil?
assert_nil(strategy.try(method_name)) assert_nil(actual_value)
else else
assert_equal(expected_value, strategy.try(method_name)) assert_equal(expected_value, actual_value)
end end
end end
end end

470
test/unit/sources/deviant_art_test.rb
View File

@@ -1,4 +1,4 @@
require 'test_helper' require "test_helper"
module Sources module Sources
class DeviantArtTest < ActiveSupport::TestCase class DeviantArtTest < ActiveSupport::TestCase
@@ -7,352 +7,206 @@ module Sources
skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present? skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present?
end end
context "A page url" do context "A deviantart post" do
setup do strategy_should_work(
@site = Source::Extractor.find("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484") "https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484",
end image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/11a24395-0f24-446d-ae73-a9f812e79e55/d70rm0s-e5b6b5e6-5795-44bb-a0ba-27b5c2349be7\.jpg}],
download_size: 877_987,
should "work" do page_url: "https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484",
# http://origin-orig.deviantart.net/d533/f/2014/004/8/d/holiday_elincia_by_aeror404-d70rm0s.jpg (md5: a7651a6586b95c62fd593dd34bb13618, size: 877987) artist_name: "aeror404",
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/11a24395-0f24-446d-ae73-a9f812e79e55/d70rm0s-e5b6b5e6-5795-44bb-a0ba-27b5c2349be7\.jpg}, @site.image_urls.sole) profile_url: "https://www.deviantart.com/aeror404",
assert_downloaded(877_987, @site.image_urls.sole) artist_commentary_title: "Holiday Elincia"
)
assert_equal("aeror404", @site.artist_name)
assert_equal("https://www.deviantart.com/aeror404", @site.profile_url)
assert_equal("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484", @site.page_url)
assert_equal("Holiday Elincia", @site.artist_commentary_title)
end
end end
context "The source for a deleted DeviantArt image URL" do context "A deviantart image" do
should "work" do strategy_should_work(
@site = Source::Extractor.find("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png") "https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png",
@artist = create(:artist, name: "nickbeja", url_string: "https://nickbeja.deviantart.com") image_urls: ["https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png"],
download_size: 840_296,
assert_equal(["https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png"], @site.image_urls) page_url: "https://www.deviantart.com/nickbeja/art/Mindflayer-Girl01-708675884",
assert_equal("nickbeja", @site.artist_name) artist_name: "nickbeja",
assert_equal("https://www.deviantart.com/nickbeja", @site.profile_url) profile_url: "https://www.deviantart.com/nickbeja"
assert_equal("https://www.deviantart.com/nickbeja/art/Mindflayer-Girl01-708675884", @site.page_url) )
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end end
context "The source for a download-disabled DeviantArt artwork page" do context "Another deviantart image" do
should "get the image url" do strategy_should_work(
@site = Source::Extractor.find("https://noizave.deviantart.com/art/test-no-download-697415967") "https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/b1f96af6-56a3-47a8-b7f4-406f243af3a3/daihpha-9f1fcd2e-7557-4db5-951b-9aedca9a3ae7\.jpg}],
# https://img00.deviantart.net/56ee/i/2017/219/2/3/test__no_download_by_noizave-dbj81lr.jpg (md5: 25a03b5a6744b6b914a13b3cd50e3c2c, size: 37638) download_size: 906_621,
# orig file: https://danbooru.donmai.us/posts/463438 (md5: eb97244675e47dbd77ffcd2d7e15aeab, size: 59401) page_url: "https://www.deviantart.com/hideyoshi/art/Legend-of-Galactic-Heroes-635721022",
assert_match("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbj81lr-3306feb1-87dc-4d25-9a4c-da8d2973a8b7.jpg", @site.image_urls.sole) artist_name: "hideyoshi",
assert_downloaded(40_036, @site.image_urls.sole) profile_url: "https://www.deviantart.com/hideyoshi",
tags: %w[barbarossa bay brunhild flare hangar odin planet ship spaceship sun sunset brünhild legendsofgalacticheroes]
assert_equal("noizave", @site.artist_name) )
assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
assert_equal("test, no download", @site.artist_commentary_title)
assert_equal("https://www.deviantart.com/noizave/art/test-no-download-697415967", @site.page_url)
end
end end
context "The source for a download-enabled DeviantArt artwork page" do context "A deviantart origin-orig image" do
should "get the download image url" do desc = <<-EOS.strip_heredoc.chomp
@site = Source::Extractor.find("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781") blah blah
"test link":[http://www.google.com]
# http://origin-orig.deviantart.net/a713/f/2018/333/3/6/all_that_glitters_ii_by_len1-dct67m5.jpg (md5: d16bb8620600334caa029ebb9bc426a6, size: 1402017) h1. lol
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/a6289ca5-2205-4118-af55-c6934fba0930/dct67m5-51e8db38-9167-4f5c-931d-561ea4d3810d\.jpg}, @site.image_urls.sole)
assert_downloaded(1402017, @site.image_urls.sole)
assert_equal("len1", @site.artist_name)
assert_equal("https://www.deviantart.com/len1", @site.profile_url)
assert_equal("All that Glitters II", @site.artist_commentary_title) [b]blah[/b] [i]blah[/i] [u]blah[/u] [s]blah[/s]
assert_equal("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781", @site.page_url) herp derp
end
[quote]this is a quote[/quote]
* one
* two
* three
* one
* two
* three
"Heart":[https://e.deviantart.net/emoticons/h/heart.gif]
EOS
strategy_should_work(
"http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}],
download_size: 3_619,
page_url: "https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408",
artist_name: "noizave",
profile_url: "https://www.deviantart.com/noizave",
tags: %w[bar baz foo],
artist_commentary_title: "test post please ignore",
dtext_artist_commentary_desc: desc
)
end end
context "The source for a DeviantArt image url" do context "A img00.deviantart.net sample" do
should "fetch the source data" do strategy_should_work(
@site = Source::Extractor.find("https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg") "https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}],
# http://origin-orig.deviantart.net/9e1f/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg (md5: 4cfec3d50ebbb924077cc5c90e705d4e, size: 906621) download_size: 3_619,
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/b1f96af6-56a3-47a8-b7f4-406f243af3a3/daihpha-9f1fcd2e-7557-4db5-951b-9aedca9a3ae7\.jpg}, @site.image_urls.sole) page_url: "https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408"
assert_downloaded(906_621, @site.image_urls.sole) )
assert_equal("hideyoshi", @site.artist_name)
assert_equal("https://www.deviantart.com/hideyoshi", @site.profile_url)
assert_equal("https://www.deviantart.com/hideyoshi/art/Legend-of-Galactic-Heroes-635721022", @site.page_url)
assert_equal(%w[barbarossa bay brunhild flare hangar odin planet ship spaceship sun sunset brünhild legendsofgalacticheroes].sort, @site.tags.map(&:first).sort)
end
end end
context "The source for a origin-orig.deviantart.net image url without a referer" do context "A th00.deviantart.net/*/PRE/* thumbnail" do
should "work" do strategy_should_work(
@site = Source::Extractor.find("http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png") "http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png",
image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/d8995973-0b32-4a7d-8cd8-d847d083689a/d797tit-1eac22e0-38b6-4eae-adcb-1b72843fd62a\.png}],
# md5: 9dec050536dbdb09ab63cb9c5a48f8b7 download_size: 3_391_584,
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole) page_url: "https://www.deviantart.com/xyelkiltrox/art/Goruto-438744629"
assert_downloaded(3619, @site.image_urls.sole) )
assert_equal("https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", @site.page_url)
assert_equal("noizave", @site.artist_name)
assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
assert_equal(%w[bar baz foo], @site.tags.map(&:first))
assert_nothing_raised { @site.to_h }
end
end end
context "The source for a img00.deviantart.net sample image url" do context "A deviantart page with download disabled" do
should "return the full size image url" do strategy_should_work(
@site = Source::Extractor.find("https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png") "https://noizave.deviantart.com/art/test-no-download-697415967",
image_urls: [%r{https://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbj81lr-3306feb1-87dc-4d25-9a4c-da8d2973a8b7\.jpg\?token=}],
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole) download_size: 59_401,
assert_downloaded(3619, @site.image_urls.sole) page_url: "https://www.deviantart.com/noizave/art/test-no-download-697415967",
artist_name: "noizave",
assert_equal("https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", @site.page_url) profile_url: "https://www.deviantart.com/noizave",
end artist_commentary_title: "test, no download"
)
end end
context "The source for a th00.deviantart.net/*/PRE/* thumbnail url" do context "A deviantart page with download disabled for a huge file" do
should "return the full size image url" do strategy_should_work(
@site = Source::Extractor.find("http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png") "https://www.deviantart.com/anatofinnstark/art/The-Blade-of-Miquella-914166242",
download_size: 26_037_561
# http://origin-orig.deviantart.net/0f1e/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png (md5: d779f5a7da29ec90d777a8db38d07994, size: 3391584) )
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/d8995973-0b32-4a7d-8cd8-d847d083689a/d797tit-1eac22e0-38b6-4eae-adcb-1b72843fd62a\.png}, @site.image_urls.sole)
assert_downloaded(3_391_584, @site.image_urls.sole)
assert_equal("https://www.deviantart.com/xyelkiltrox/art/Goruto-438744629", @site.page_url)
end
end end
context "A source for a *.deviantart.net/*/:title_by_:artist.jpg url artist name containing underscores" do context "A deviantart page with download enabled" do
should "find the correct artist" do strategy_should_work(
@site = Source::Extractor.find("https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg") "https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781",
@artist = create(:artist, name: "mikoto-chan", url_string: "https://www.deviantart.com/mikoto-chan") image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/a6289ca5-2205-4118-af55-c6934fba0930/dct67m5-51e8db38-9167-4f5c-931d-561ea4d3810d\.jpg}],
download_size: 1_402_017,
assert_equal("mikoto-chan", @site.artist_name) page_url: "https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781",
assert_equal([@artist], @site.artists) artist_name: "len1",
assert_nil(@site.page_url) profile_url: "https://www.deviantart.com/len1",
end artist_commentary_title: "All that Glitters II"
)
end end
context "The source for a *.deviantart.net/*/:title_by_:artist.jpg url" do context "A *.deviantart.net/*/:title_by_:artist.jpg image with an artist name containing underscores" do
setup do strategy_should_work(
@url = "http://fc08.deviantart.net/files/f/2007/120/c/9/cool_like_me_by_47ness.jpg" "https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg",
@ref = "https://47ness.deviantart.com/art/Cool-Like-Me-54339311" image_urls: ["https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg"],
@artist = create(:artist, name: "47ness", url_string: "https://www.deviantart.com/47ness") artist_name: "mikoto-chan",
end profile_url: "https://www.deviantart.com/mikoto-chan",
page_url: nil
context "without a referer" do )
should "work" do
@site = Source::Extractor.find(@url)
assert_equal([@site.url], @site.image_urls)
assert_equal("47ness", @site.artist_name)
assert_equal("https://www.deviantart.com/47ness", @site.profile_url)
assert_nil(@site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
context "with a referer" do
should "work" do
@site = Source::Extractor.find(@url, @ref)
# http://origin-orig.deviantart.net/a418/f/2007/120/c/9/cool_like_me_by_47ness.jpg (md5: da78e7c192d42470acda7d87ade64849, size: 265496)
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ece2238f-5c8f-48e4-afda-304cab294acd/dwcohb-8189be91-691d-4212-b3a0-0b77e86a57d1\.jpg}, @site.image_urls.sole)
assert_downloaded(265_496, @site.image_urls.sole)
assert_equal("47ness", @site.artist_name)
assert_equal("https://www.deviantart.com/47ness", @site.profile_url)
assert_equal("https://www.deviantart.com/47ness/art/Cool-Like-Me-54339311", @site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
end end
context "The source for a *.deviantart.net/*/:hash.jpg url" do context "A *.deviantart.net/*/:hash.jpg image without referer" do
setup do strategy_should_work(
@url = "http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg" "http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg",
@ref = "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896" image_urls: ["http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg"],
@artist = create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com") profile_url: nil,
end page_url: nil
)
context "without a referer" do
should "work" do
@site = Source::Extractor.find(@url)
assert_equal([@url], @site.image_urls)
assert_nil(@site.artist_name)
assert_nil(@site.profile_url)
assert_nil(@site.page_url)
assert_equal([], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
context "with a referer" do
should "work" do
@site = Source::Extractor.find(@url, @ref)
assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_urls.sole)
assert_equal("edsfox", @site.artist_name)
assert_equal("https://www.deviantart.com/edsfox", @site.profile_url)
assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", @site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
end end
context "The source for a images-wixmp-.* sample image" do context "A *.deviantart.net/*/:hash.jpg image with referer" do
setup do strategy_should_work(
@url = "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg" "http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg",
@ref = "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896" referer: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896",
@artist = create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com") image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg}],
end artist_name: "edsfox",
profile_url: "https://www.deviantart.com/edsfox",
context "with a referer" do page_url: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896"
should "work" do )
@site = Source::Extractor.find(@url, @ref)
assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_urls.sole)
assert_equal("edsfox", @site.artist_name)
assert_equal("https://www.deviantart.com/edsfox", @site.profile_url)
assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", @site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
end end
context "The source for a api-da.wixmp.com image" do context "A images-wixmp-.* sample" do
setup do strategy_should_work(
@url = "https://api-da.wixmp.com/_api/download/file?downloadToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsImV4cCI6MTU5MDkwMTUzMywiaWF0IjoxNTkwOTAwOTIzLCJqdGkiOiI1ZWQzMzhjNWQ5YjI0Iiwib2JqIjpudWxsLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdLCJwYXlsb2FkIjp7InBhdGgiOiJcL2ZcL2U0NmE0OGViLTNkMGItNDQ5ZS05MGRjLTBhMWIzMWNiMTM2MVwvZGQzcDF4OS1mYjQ3YmM4Zi02NTNlLTQyYTItYmI0ZC1hZmFmOWZjMmI3ODEuanBnIn19.-zo8E2eDmkmDNCK-sMabBajkaGtVYJ2Q20iVrUtt05Q" "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg",
@ref = "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029" referer: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896",
@artist = create(:artist, name: "akizero", url_string: "https://akizero1510.deviantart.com") image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg}],
end artist_name: "edsfox",
profile_url: "https://www.deviantart.com/edsfox",
context "with a referer" do page_url: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896"
should "work" do )
@site = Source::Extractor.find(@url, @ref)
assert_equal(@ref, @site.page_url)
assert_equal([@artist], @site.artists)
assert_nothing_raised { @site.to_h }
end
end
end end
context "The source for a non-downloadable animated gif with id<=790677560" do context "A api-da.wixmp.com sample" do
should "return working image url" do strategy_should_work(
@site = Source::Extractor.find("https://www.deviantart.com/heartgear/art/Silent-Night-579982816") "https://api-da.wixmp.com/_api/download/file?downloadToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsImV4cCI6MTU5MDkwMTUzMywiaWF0IjoxNTkwOTAwOTIzLCJqdGkiOiI1ZWQzMzhjNWQ5YjI0Iiwib2JqIjpudWxsLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdLCJwYXlsb2FkIjp7InBhdGgiOiJcL2ZcL2U0NmE0OGViLTNkMGItNDQ5ZS05MGRjLTBhMWIzMWNiMTM2MVwvZGQzcDF4OS1mYjQ3YmM4Zi02NTNlLTQyYTItYmI0ZC1hZmFmOWZjMmI3ODEuanBnIn19.-zo8E2eDmkmDNCK-sMabBajkaGtVYJ2Q20iVrUtt05Q",
referer: "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029",
# md5: 62caac1863aa264a56d548b4b7607097 page_url: "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029"
assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ea95be00-c5aa-4063-bd55-f5a9183912f7/d9lb1ls-7d625444-0003-4123-bf00-274737ca7fdd.gif\?token=!, @site.image_urls.sole) )
assert_downloaded(350_156, @site.image_urls.sole)
end
end end
context "The source for a non-downloadable flash file" do context "A non-downloadable animated gif with id<=790677560" do
should "return working image url" do strategy_should_work(
skip "https://www.deviantart.com/heartgear/art/Silent-Night-579982816",
@site = Source::Extractor.find("https://www.deviantart.com/heartgear/art/SL-40v3-522007633") image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ea95be00-c5aa-4063-bd55-f5a9183912f7/d9lb1ls-7d625444-0003-4123-bf00-274737ca7fdd.gif\?token=}],
download_size: 350_156
# md5: 6adf1a3d532f898f44cf9948cbc7db7d )
assert_match(%r!\Ahttps://api-da\.wixmp\.com/_api/download/file\?downloadToken=!, @site.image_urls.sole)
assert_downloaded(3_496_110, @site.image_urls.sole)
end
end end
context "The source for a non-downloadable video file" do context "A non-downloadable video file" do
should "return working image url" do strategy_should_work(
@site = Source::Extractor.find("https://www.deviantart.com/gs-mantis/art/Chen-Goes-Fishing-505847233") "https://www.deviantart.com/gs-mantis/art/Chen-Goes-Fishing-505847233",
image_urls: ["https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/v/mp4/fe046bc7-4d68-4699-96c1-19aa464edff6/d8d6281-91959e92-214f-4b2d-a138-ace09f4b6d09.1080p.8e57939eba634743a9fa41185e398d00.mp4"],
# md5: 344ac2b9fd5a87982af4b648aa2b2b0d download_size: 9_739_947
assert_equal(["https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/v/mp4/fe046bc7-4d68-4699-96c1-19aa464edff6/d8d6281-91959e92-214f-4b2d-a138-ace09f4b6d09.1080p.8e57939eba634743a9fa41185e398d00.mp4"], @site.image_urls) )
assert_downloaded(9_739_947, @site.image_urls.sole)
end
end end
context "The source for an DeviantArt artwork page" do context "A login-only deviantart post" do
setup do strategy_should_work(
@site = Source::Extractor.find("http://noizave.deviantart.com/art/test-post-please-ignore-685436408") "http://noizave.deviantart.com/art/hidden-work-685458369",
end image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3r29-10c99118-5cfe-4402-ad55-7b57e7c0ca43\.png}],
download_size: 3_619
should "get the image url" do )
# https://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole)
assert_downloaded(3619, @site.image_urls.sole)
end
should "get the profile" do
assert_equal("https://www.deviantart.com/noizave", @site.profile_url)
end
should "get the artist name" do
assert_equal("noizave", @site.artist_name)
end
should "get the tags" do
assert_equal(%w[bar baz foo], @site.tags.map(&:first))
end
should "get the artist commentary" do
title = "test post please ignore"
desc = "<div align=\"center\"><span>blah blah<br /><div align=\"left\"><a class=\"external\" href=\"https://www.deviantart.com/users/outgoing?http://www.google.com\">test link</a><br /></div></span></div><br /><h1>lol</h1><br /><br /><b>blah</b>&nbsp;<i>blah</i>&nbsp;<u>blah</u>&nbsp;<strike>blah</strike><br />herp derp<br /><br /><blockquote>this is a quote</blockquote><ol><li>one</li><li>two</li><li>three</li></ol><ul><li>one</li><li>two</li><li>three</li></ul><img src=\"https://e.deviantart.net/emoticons/h/heart.gif\" alt=\"Heart\" style=\"width: 15px; height: 13px;\" data-embed-type=\"emoticon\" data-embed-id=\"357\">&nbsp;&nbsp;"
assert_equal(title, @site.artist_commentary_title)
assert_equal(desc, @site.artist_commentary_desc)
end
should "get the dtext-ified commentary" do
desc = <<-EOS.strip_heredoc.chomp
blah blah
"test link":[http://www.google.com]
h1. lol
[b]blah[/b] [i]blah[/i] [u]blah[/u] [s]blah[/s]
herp derp
[quote]this is a quote[/quote]
* one
* two
* three
* one
* two
* three
"Heart":[https://e.deviantart.net/emoticons/h/heart.gif]
EOS
assert_equal(desc, @site.dtext_artist_commentary_desc)
end
end
context "The source for a login-only DeviantArt artwork page" do
setup do
@site = Source::Extractor.find("http://noizave.deviantart.com/art/hidden-work-685458369")
end
should "get the image url" do
# https://origin-orig.deviantart.net/cb25/f/2017/160/1/9/hidden_work_by_noizave-dbc3r29.png
assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3r29-10c99118-5cfe-4402-ad55-7b57e7c0ca43\.png}, @site.image_urls.sole)
assert_downloaded(3619, @site.image_urls.sole)
end
end end
context "A source with malformed links in the artist commentary" do context "A source with malformed links in the artist commentary" do
should "fix the links" do should "fix the links" do
@site = Source::Extractor.find("https://teemutaiga.deviantart.com/art/Kisu-620666655") @site = Source::Extractor.find("https://teemutaiga.deviantart.com/art/Kisu-620666655")
assert_match(%r!"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]!, @site.dtext_artist_commentary_desc) assert_match(%r{"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]}, @site.dtext_artist_commentary_desc)
end end
end end
@@ -393,7 +247,7 @@ module Sources
assert(Source::URL.page_url?("https://sta.sh/0wxs31o7nn2")) assert(Source::URL.page_url?("https://sta.sh/0wxs31o7nn2"))
assert(Source::URL.profile_url?("https://www.deviantart.com/noizave")) assert(Source::URL.profile_url?("https://www.deviantart.com/noizave"))
assert(Source::URL.profile_url?("https://noizave.deviantart.com")) assert(Source::URL.profile_url?("https://noizave.deviantart.com"))
refute(Source::URL.profile_url?("https://deviantart.net")) assert_not(Source::URL.profile_url?("https://deviantart.net"))
end end
end end
end end