diff --git a/app/logical/source/extractor/deviant_art.rb b/app/logical/source/extractor/deviant_art.rb index b7987ea70..49565f339 100644 --- a/app/logical/source/extractor/deviant_art.rb +++ b/app/logical/source/extractor/deviant_art.rb @@ -18,7 +18,11 @@ module Source def image_url # work is private, deleted, or the url didn't contain a deviation id; use image url as given by user. if api_deviation.blank? - url + if url =~ %r{\Ahttps://images-wixmp-} + extract_largest(url) + else + url + end elsif api_deviation[:is_downloadable] api_download[:src] elsif api_deviation[:flash].present? @@ -27,13 +31,28 @@ module Source api_deviation[:videos].max_by { |x| x[:filesize] }[:src] else src = api_deviation.dig(:content, :src) - if deviation_id && deviation_id.to_i <= 790_677_560 && src =~ %r{\Ahttps://images-wixmp-} && src !~ /\.gif\?/ - src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1') - src = src.sub(%r{/v1/(fit|fill)/.*\z}i, "") + extract_largest(src) + end + end + + def extract_largest(src) + if src =~ %r{\Ahttps://images-wixmp-} + sample, separator, * = src.partition("/v1/") + if separator.blank? + src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1') unless src =~ /\.gif\?/ + src + else + # :^) https://i.imgur.com/KG5bVRU.png + # shamelessly aped from: + # https://github.com/mikf/gallery-dl/blob/7990fe84f11271bc8e4079db6b0248dbeb79474a/gallery_dl/extractor/deviantart.py#L293 + *, f_value = sample.split("/f/") + data = {sub: "urn:app:", iss: "urn:app:", obj: [[{path: "/f/#{f_value}"}]], aud: ["urn:service:file.download"]} + token = Base64.encode64(JSON.generate(data)).gsub("=", "").gsub("\n", "") + "#{sample}?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.#{token}." end + else src = src.sub(%r{\Ahttps?://orig\d+\.deviantart\.net}i, "http://origin-orig.deviantart.net") - src = src.gsub(/q_\d+,strp/, "q_100") - src + src.gsub(/q_\d+,strp/, "q_100") end end diff --git a/test/test_helpers/source_test_helper.rb b/test/test_helpers/source_test_helper.rb index c930b85c6..c4bb83d88 100644 --- a/test/test_helpers/source_test_helper.rb +++ b/test/test_helpers/source_test_helper.rb @@ -86,15 +86,22 @@ module SourceTestHelper # check any method that is passed as kwargs, in order to hardcode as few things as possible # XXX can't use **kwargs because of a bug with shoulda-context, so we're using a hash temporarily methods_to_test.each do |method_name, expected_value| + actual_value = strategy.try(method_name) + should "make sure that '#{method_name}' matches" do if expected_value.instance_of? Regexp - assert_match(expected_value, strategy.try(method_name)) + assert_match(expected_value, actual_value) elsif expected_value.instance_of? Array - assert_equal(expected_value.sort, strategy.try(method_name).sort) + if expected_value.first.instance_of? Regexp + actual_values = actual_value.sort + expected_value.sort.each_with_index { |each_value, index| assert_match(each_value, actual_values[index]) } + else + assert_equal(expected_value.sort, actual_value) + end elsif expected_value.nil? - assert_nil(strategy.try(method_name)) + assert_nil(actual_value) else - assert_equal(expected_value, strategy.try(method_name)) + assert_equal(expected_value, actual_value) end end end diff --git a/test/unit/sources/deviant_art_test.rb b/test/unit/sources/deviant_art_test.rb index 719fa2f4a..548feb336 100644 --- a/test/unit/sources/deviant_art_test.rb +++ b/test/unit/sources/deviant_art_test.rb @@ -1,4 +1,4 @@ -require 'test_helper' +require "test_helper" module Sources class DeviantArtTest < ActiveSupport::TestCase @@ -7,352 +7,206 @@ module Sources skip "DeviantArt API keys not set" unless Danbooru.config.deviantart_client_id.present? end - context "A page url" do - setup do - @site = Source::Extractor.find("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484") - end - - should "work" do - # http://origin-orig.deviantart.net/d533/f/2014/004/8/d/holiday_elincia_by_aeror404-d70rm0s.jpg (md5: a7651a6586b95c62fd593dd34bb13618, size: 877987) - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/11a24395-0f24-446d-ae73-a9f812e79e55/d70rm0s-e5b6b5e6-5795-44bb-a0ba-27b5c2349be7\.jpg}, @site.image_urls.sole) - assert_downloaded(877_987, @site.image_urls.sole) - - assert_equal("aeror404", @site.artist_name) - assert_equal("https://www.deviantart.com/aeror404", @site.profile_url) - assert_equal("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484", @site.page_url) - assert_equal("Holiday Elincia", @site.artist_commentary_title) - end + context "A deviantart post" do + strategy_should_work( + "https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484", + image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/11a24395-0f24-446d-ae73-a9f812e79e55/d70rm0s-e5b6b5e6-5795-44bb-a0ba-27b5c2349be7\.jpg}], + download_size: 877_987, + page_url: "https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484", + artist_name: "aeror404", + profile_url: "https://www.deviantart.com/aeror404", + artist_commentary_title: "Holiday Elincia" + ) end - context "The source for a deleted DeviantArt image URL" do - should "work" do - @site = Source::Extractor.find("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png") - @artist = create(:artist, name: "nickbeja", url_string: "https://nickbeja.deviantart.com") - - assert_equal(["https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png"], @site.image_urls) - assert_equal("nickbeja", @site.artist_name) - assert_equal("https://www.deviantart.com/nickbeja", @site.profile_url) - assert_equal("https://www.deviantart.com/nickbeja/art/Mindflayer-Girl01-708675884", @site.page_url) - assert_equal([@artist], @site.artists) - assert_nothing_raised { @site.to_h } - end + context "A deviantart image" do + strategy_should_work( + "https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png", + image_urls: ["https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png"], + download_size: 840_296, + page_url: "https://www.deviantart.com/nickbeja/art/Mindflayer-Girl01-708675884", + artist_name: "nickbeja", + profile_url: "https://www.deviantart.com/nickbeja" + ) end - context "The source for a download-disabled DeviantArt artwork page" do - should "get the image url" do - @site = Source::Extractor.find("https://noizave.deviantart.com/art/test-no-download-697415967") - - # https://img00.deviantart.net/56ee/i/2017/219/2/3/test__no_download_by_noizave-dbj81lr.jpg (md5: 25a03b5a6744b6b914a13b3cd50e3c2c, size: 37638) - # orig file: https://danbooru.donmai.us/posts/463438 (md5: eb97244675e47dbd77ffcd2d7e15aeab, size: 59401) - assert_match("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbj81lr-3306feb1-87dc-4d25-9a4c-da8d2973a8b7.jpg", @site.image_urls.sole) - assert_downloaded(40_036, @site.image_urls.sole) - - assert_equal("noizave", @site.artist_name) - assert_equal("https://www.deviantart.com/noizave", @site.profile_url) - assert_equal("test, no download", @site.artist_commentary_title) - assert_equal("https://www.deviantart.com/noizave/art/test-no-download-697415967", @site.page_url) - end + context "Another deviantart image" do + strategy_should_work( + "https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg", + image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/b1f96af6-56a3-47a8-b7f4-406f243af3a3/daihpha-9f1fcd2e-7557-4db5-951b-9aedca9a3ae7\.jpg}], + download_size: 906_621, + page_url: "https://www.deviantart.com/hideyoshi/art/Legend-of-Galactic-Heroes-635721022", + artist_name: "hideyoshi", + profile_url: "https://www.deviantart.com/hideyoshi", + tags: %w[barbarossa bay brunhild flare hangar odin planet ship spaceship sun sunset brünhild legendsofgalacticheroes] + ) end - context "The source for a download-enabled DeviantArt artwork page" do - should "get the download image url" do - @site = Source::Extractor.find("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781") + context "A deviantart origin-orig image" do + desc = <<-EOS.strip_heredoc.chomp + blah blah + "test link":[http://www.google.com] - # http://origin-orig.deviantart.net/a713/f/2018/333/3/6/all_that_glitters_ii_by_len1-dct67m5.jpg (md5: d16bb8620600334caa029ebb9bc426a6, size: 1402017) - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/a6289ca5-2205-4118-af55-c6934fba0930/dct67m5-51e8db38-9167-4f5c-931d-561ea4d3810d\.jpg}, @site.image_urls.sole) - assert_downloaded(1402017, @site.image_urls.sole) + h1. lol - assert_equal("len1", @site.artist_name) - assert_equal("https://www.deviantart.com/len1", @site.profile_url) - assert_equal("All that Glitters II", @site.artist_commentary_title) - assert_equal("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781", @site.page_url) - end + + + [b]blah[/b] [i]blah[/i] [u]blah[/u] [s]blah[/s] + herp derp + + [quote]this is a quote[/quote] + + * one + * two + * three + + * one + * two + * three + + "Heart":[https://e.deviantart.net/emoticons/h/heart.gif] + EOS + + strategy_should_work( + "http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png", + image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}], + download_size: 3_619, + page_url: "https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", + artist_name: "noizave", + profile_url: "https://www.deviantart.com/noizave", + tags: %w[bar baz foo], + artist_commentary_title: "test post please ignore", + dtext_artist_commentary_desc: desc + ) end - context "The source for a DeviantArt image url" do - should "fetch the source data" do - @site = Source::Extractor.find("https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg") - - # http://origin-orig.deviantart.net/9e1f/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg (md5: 4cfec3d50ebbb924077cc5c90e705d4e, size: 906621) - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/b1f96af6-56a3-47a8-b7f4-406f243af3a3/daihpha-9f1fcd2e-7557-4db5-951b-9aedca9a3ae7\.jpg}, @site.image_urls.sole) - assert_downloaded(906_621, @site.image_urls.sole) - - assert_equal("hideyoshi", @site.artist_name) - assert_equal("https://www.deviantart.com/hideyoshi", @site.profile_url) - assert_equal("https://www.deviantart.com/hideyoshi/art/Legend-of-Galactic-Heroes-635721022", @site.page_url) - assert_equal(%w[barbarossa bay brunhild flare hangar odin planet ship spaceship sun sunset brünhild legendsofgalacticheroes].sort, @site.tags.map(&:first).sort) - end + context "A img00.deviantart.net sample" do + strategy_should_work( + "https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png", + image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}], + download_size: 3_619, + page_url: "https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408" + ) end - context "The source for a origin-orig.deviantart.net image url without a referer" do - should "work" do - @site = Source::Extractor.find("http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png") - - # md5: 9dec050536dbdb09ab63cb9c5a48f8b7 - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole) - assert_downloaded(3619, @site.image_urls.sole) - - assert_equal("https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", @site.page_url) - - assert_equal("noizave", @site.artist_name) - assert_equal("https://www.deviantart.com/noizave", @site.profile_url) - assert_equal(%w[bar baz foo], @site.tags.map(&:first)) - assert_nothing_raised { @site.to_h } - end + context "A th00.deviantart.net/*/PRE/* thumbnail" do + strategy_should_work( + "http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png", + image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/d8995973-0b32-4a7d-8cd8-d847d083689a/d797tit-1eac22e0-38b6-4eae-adcb-1b72843fd62a\.png}], + download_size: 3_391_584, + page_url: "https://www.deviantart.com/xyelkiltrox/art/Goruto-438744629" + ) end - context "The source for a img00.deviantart.net sample image url" do - should "return the full size image url" do - @site = Source::Extractor.find("https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png") - - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole) - assert_downloaded(3619, @site.image_urls.sole) - - assert_equal("https://www.deviantart.com/noizave/art/test-post-please-ignore-685436408", @site.page_url) - end + context "A deviantart page with download disabled" do + strategy_should_work( + "https://noizave.deviantart.com/art/test-no-download-697415967", + image_urls: [%r{https://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbj81lr-3306feb1-87dc-4d25-9a4c-da8d2973a8b7\.jpg\?token=}], + download_size: 59_401, + page_url: "https://www.deviantart.com/noizave/art/test-no-download-697415967", + artist_name: "noizave", + profile_url: "https://www.deviantart.com/noizave", + artist_commentary_title: "test, no download" + ) end - context "The source for a th00.deviantart.net/*/PRE/* thumbnail url" do - should "return the full size image url" do - @site = Source::Extractor.find("http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png") - - # http://origin-orig.deviantart.net/0f1e/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png (md5: d779f5a7da29ec90d777a8db38d07994, size: 3391584) - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/d8995973-0b32-4a7d-8cd8-d847d083689a/d797tit-1eac22e0-38b6-4eae-adcb-1b72843fd62a\.png}, @site.image_urls.sole) - assert_downloaded(3_391_584, @site.image_urls.sole) - - assert_equal("https://www.deviantart.com/xyelkiltrox/art/Goruto-438744629", @site.page_url) - end + context "A deviantart page with download disabled for a huge file" do + strategy_should_work( + "https://www.deviantart.com/anatofinnstark/art/The-Blade-of-Miquella-914166242", + download_size: 26_037_561 + ) end - context "A source for a *.deviantart.net/*/:title_by_:artist.jpg url artist name containing underscores" do - should "find the correct artist" do - @site = Source::Extractor.find("https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg") - @artist = create(:artist, name: "mikoto-chan", url_string: "https://www.deviantart.com/mikoto-chan") - - assert_equal("mikoto-chan", @site.artist_name) - assert_equal([@artist], @site.artists) - assert_nil(@site.page_url) - end + context "A deviantart page with download enabled" do + strategy_should_work( + "https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781", + image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/a6289ca5-2205-4118-af55-c6934fba0930/dct67m5-51e8db38-9167-4f5c-931d-561ea4d3810d\.jpg}], + download_size: 1_402_017, + page_url: "https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781", + artist_name: "len1", + profile_url: "https://www.deviantart.com/len1", + artist_commentary_title: "All that Glitters II" + ) end - context "The source for a *.deviantart.net/*/:title_by_:artist.jpg url" do - setup do - @url = "http://fc08.deviantart.net/files/f/2007/120/c/9/cool_like_me_by_47ness.jpg" - @ref = "https://47ness.deviantart.com/art/Cool-Like-Me-54339311" - @artist = create(:artist, name: "47ness", url_string: "https://www.deviantart.com/47ness") - end - - context "without a referer" do - should "work" do - @site = Source::Extractor.find(@url) - - assert_equal([@site.url], @site.image_urls) - assert_equal("47ness", @site.artist_name) - assert_equal("https://www.deviantart.com/47ness", @site.profile_url) - assert_nil(@site.page_url) - assert_equal([@artist], @site.artists) - assert_nothing_raised { @site.to_h } - end - end - - context "with a referer" do - should "work" do - @site = Source::Extractor.find(@url, @ref) - - # http://origin-orig.deviantart.net/a418/f/2007/120/c/9/cool_like_me_by_47ness.jpg (md5: da78e7c192d42470acda7d87ade64849, size: 265496) - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ece2238f-5c8f-48e4-afda-304cab294acd/dwcohb-8189be91-691d-4212-b3a0-0b77e86a57d1\.jpg}, @site.image_urls.sole) - assert_downloaded(265_496, @site.image_urls.sole) - - assert_equal("47ness", @site.artist_name) - assert_equal("https://www.deviantart.com/47ness", @site.profile_url) - assert_equal("https://www.deviantart.com/47ness/art/Cool-Like-Me-54339311", @site.page_url) - assert_equal([@artist], @site.artists) - assert_nothing_raised { @site.to_h } - end - end + context "A *.deviantart.net/*/:title_by_:artist.jpg image with an artist name containing underscores" do + strategy_should_work( + "https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg", + image_urls: ["https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg"], + artist_name: "mikoto-chan", + profile_url: "https://www.deviantart.com/mikoto-chan", + page_url: nil + ) end - context "The source for a *.deviantart.net/*/:hash.jpg url" do - setup do - @url = "http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg" - @ref = "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896" - @artist = create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com") - end - - context "without a referer" do - should "work" do - @site = Source::Extractor.find(@url) - - assert_equal([@url], @site.image_urls) - assert_nil(@site.artist_name) - assert_nil(@site.profile_url) - assert_nil(@site.page_url) - assert_equal([], @site.artists) - assert_nothing_raised { @site.to_h } - end - end - - context "with a referer" do - should "work" do - @site = Source::Extractor.find(@url, @ref) - - assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_urls.sole) - assert_equal("edsfox", @site.artist_name) - assert_equal("https://www.deviantart.com/edsfox", @site.profile_url) - assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", @site.page_url) - assert_equal([@artist], @site.artists) - assert_nothing_raised { @site.to_h } - end - end + context "A *.deviantart.net/*/:hash.jpg image without referer" do + strategy_should_work( + "http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg", + image_urls: ["http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg"], + profile_url: nil, + page_url: nil + ) end - context "The source for a images-wixmp-.* sample image" do - setup do - @url = "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg" - @ref = "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896" - @artist = create(:artist, name: "edsfox", url_string: "https://edsfox.deviantart.com") - end - - context "with a referer" do - should "work" do - @site = Source::Extractor.find(@url, @ref) - - assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_urls.sole) - assert_equal("edsfox", @site.artist_name) - assert_equal("https://www.deviantart.com/edsfox", @site.profile_url) - assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", @site.page_url) - assert_equal([@artist], @site.artists) - assert_nothing_raised { @site.to_h } - end - end + context "A *.deviantart.net/*/:hash.jpg image with referer" do + strategy_should_work( + "http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg", + referer: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", + image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg}], + artist_name: "edsfox", + profile_url: "https://www.deviantart.com/edsfox", + page_url: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896" + ) end - context "The source for a api-da.wixmp.com image" do - setup do - @url = "https://api-da.wixmp.com/_api/download/file?downloadToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsImV4cCI6MTU5MDkwMTUzMywiaWF0IjoxNTkwOTAwOTIzLCJqdGkiOiI1ZWQzMzhjNWQ5YjI0Iiwib2JqIjpudWxsLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdLCJwYXlsb2FkIjp7InBhdGgiOiJcL2ZcL2U0NmE0OGViLTNkMGItNDQ5ZS05MGRjLTBhMWIzMWNiMTM2MVwvZGQzcDF4OS1mYjQ3YmM4Zi02NTNlLTQyYTItYmI0ZC1hZmFmOWZjMmI3ODEuanBnIn19.-zo8E2eDmkmDNCK-sMabBajkaGtVYJ2Q20iVrUtt05Q" - @ref = "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029" - @artist = create(:artist, name: "akizero", url_string: "https://akizero1510.deviantart.com") - end - - context "with a referer" do - should "work" do - @site = Source::Extractor.find(@url, @ref) - - assert_equal(@ref, @site.page_url) - assert_equal([@artist], @site.artists) - assert_nothing_raised { @site.to_h } - end - end + context "A images-wixmp-.* sample" do + strategy_should_work( + "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg", + referer: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", + image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg}], + artist_name: "edsfox", + profile_url: "https://www.deviantart.com/edsfox", + page_url: "https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896" + ) end - context "The source for a non-downloadable animated gif with id<=790677560" do - should "return working image url" do - @site = Source::Extractor.find("https://www.deviantart.com/heartgear/art/Silent-Night-579982816") - - # md5: 62caac1863aa264a56d548b4b7607097 - assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ea95be00-c5aa-4063-bd55-f5a9183912f7/d9lb1ls-7d625444-0003-4123-bf00-274737ca7fdd.gif\?token=!, @site.image_urls.sole) - assert_downloaded(350_156, @site.image_urls.sole) - end + context "A api-da.wixmp.com sample" do + strategy_should_work( + "https://api-da.wixmp.com/_api/download/file?downloadToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsImV4cCI6MTU5MDkwMTUzMywiaWF0IjoxNTkwOTAwOTIzLCJqdGkiOiI1ZWQzMzhjNWQ5YjI0Iiwib2JqIjpudWxsLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdLCJwYXlsb2FkIjp7InBhdGgiOiJcL2ZcL2U0NmE0OGViLTNkMGItNDQ5ZS05MGRjLTBhMWIzMWNiMTM2MVwvZGQzcDF4OS1mYjQ3YmM4Zi02NTNlLTQyYTItYmI0ZC1hZmFmOWZjMmI3ODEuanBnIn19.-zo8E2eDmkmDNCK-sMabBajkaGtVYJ2Q20iVrUtt05Q", + referer: "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029", + page_url: "https://www.deviantart.com/akizero1510/art/Ten-miles-of-cherry-blossoms-792268029" + ) end - context "The source for a non-downloadable flash file" do - should "return working image url" do - skip - @site = Source::Extractor.find("https://www.deviantart.com/heartgear/art/SL-40v3-522007633") - - # md5: 6adf1a3d532f898f44cf9948cbc7db7d - assert_match(%r!\Ahttps://api-da\.wixmp\.com/_api/download/file\?downloadToken=!, @site.image_urls.sole) - assert_downloaded(3_496_110, @site.image_urls.sole) - end + context "A non-downloadable animated gif with id<=790677560" do + strategy_should_work( + "https://www.deviantart.com/heartgear/art/Silent-Night-579982816", + image_urls: [%r{\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ea95be00-c5aa-4063-bd55-f5a9183912f7/d9lb1ls-7d625444-0003-4123-bf00-274737ca7fdd.gif\?token=}], + download_size: 350_156 + ) end - context "The source for a non-downloadable video file" do - should "return working image url" do - @site = Source::Extractor.find("https://www.deviantart.com/gs-mantis/art/Chen-Goes-Fishing-505847233") - - # md5: 344ac2b9fd5a87982af4b648aa2b2b0d - assert_equal(["https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/v/mp4/fe046bc7-4d68-4699-96c1-19aa464edff6/d8d6281-91959e92-214f-4b2d-a138-ace09f4b6d09.1080p.8e57939eba634743a9fa41185e398d00.mp4"], @site.image_urls) - assert_downloaded(9_739_947, @site.image_urls.sole) - end + context "A non-downloadable video file" do + strategy_should_work( + "https://www.deviantart.com/gs-mantis/art/Chen-Goes-Fishing-505847233", + image_urls: ["https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/v/mp4/fe046bc7-4d68-4699-96c1-19aa464edff6/d8d6281-91959e92-214f-4b2d-a138-ace09f4b6d09.1080p.8e57939eba634743a9fa41185e398d00.mp4"], + download_size: 9_739_947 + ) end - context "The source for an DeviantArt artwork page" do - setup do - @site = Source::Extractor.find("http://noizave.deviantart.com/art/test-post-please-ignore-685436408") - end - - should "get the image url" do - # https://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole) - assert_downloaded(3619, @site.image_urls.sole) - end - - should "get the profile" do - assert_equal("https://www.deviantart.com/noizave", @site.profile_url) - end - - should "get the artist name" do - assert_equal("noizave", @site.artist_name) - end - - should "get the tags" do - assert_equal(%w[bar baz foo], @site.tags.map(&:first)) - end - - should "get the artist commentary" do - title = "test post please ignore" - desc = "
blah blah
test link

lol



blah blah blah blah
herp derp

this is a quote
  1. one
  2. two
  3. three
\"Heart\"  " - - assert_equal(title, @site.artist_commentary_title) - assert_equal(desc, @site.artist_commentary_desc) - end - - should "get the dtext-ified commentary" do - desc = <<-EOS.strip_heredoc.chomp - blah blah - "test link":[http://www.google.com] - - h1. lol - - - - [b]blah[/b] [i]blah[/i] [u]blah[/u] [s]blah[/s] - herp derp - - [quote]this is a quote[/quote] - - * one - * two - * three - - * one - * two - * three - - "Heart":[https://e.deviantart.net/emoticons/h/heart.gif] - EOS - - assert_equal(desc, @site.dtext_artist_commentary_desc) - end - end - - context "The source for a login-only DeviantArt artwork page" do - setup do - @site = Source::Extractor.find("http://noizave.deviantart.com/art/hidden-work-685458369") - end - - should "get the image url" do - # https://origin-orig.deviantart.net/cb25/f/2017/160/1/9/hidden_work_by_noizave-dbc3r29.png - assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3r29-10c99118-5cfe-4402-ad55-7b57e7c0ca43\.png}, @site.image_urls.sole) - assert_downloaded(3619, @site.image_urls.sole) - end + context "A login-only deviantart post" do + strategy_should_work( + "http://noizave.deviantart.com/art/hidden-work-685458369", + image_urls: [%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3r29-10c99118-5cfe-4402-ad55-7b57e7c0ca43\.png}], + download_size: 3_619 + ) end context "A source with malformed links in the artist commentary" do should "fix the links" do @site = Source::Extractor.find("https://teemutaiga.deviantart.com/art/Kisu-620666655") - assert_match(%r!"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]!, @site.dtext_artist_commentary_desc) + assert_match(%r{"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]}, @site.dtext_artist_commentary_desc) end end @@ -393,7 +247,7 @@ module Sources assert(Source::URL.page_url?("https://sta.sh/0wxs31o7nn2")) assert(Source::URL.profile_url?("https://www.deviantart.com/noizave")) assert(Source::URL.profile_url?("https://noizave.deviantart.com")) - refute(Source::URL.profile_url?("https://deviantart.net")) + assert_not(Source::URL.profile_url?("https://deviantart.net")) end end end