jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor sources

Browse Source
This commit is contained in:
Albert Yi
2018-08-06 17:39:25 -07:00
parent 54363ffecf
commit 762dc3da24
71 changed files with 2340 additions and 2430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
test/unit/downloads/art_station_test.rb
View File

@@ -4,31 +4,35 @@ module Downloads
class ArtStationTest < ActiveSupport::TestCase
context "a download for a (small) artstation image" do
setup do
@source = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
@download = Downloads::File.new(@source)
@asset = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974"
@download = Downloads::File.new(@asset)
end
should "download the large image instead" do
assert_equal("https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974", @download.source)
file, strategy = @download.download!
assert_equal(517_706, ::File.size(file.path))
end
end
context "for an image where an original does not exist" do
setup do
@source = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg"
@download = Downloads::File.new(@source)
@download.download!
@asset = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg"
@download = Downloads::File.new(@asset)
end
should "not try to download the original" do
assert_equal("https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg", @download.source)
file, strategy = @download.download!
assert_equal(449_047, ::File.size(file.path))
end
end
context "a download for an ArtStation image hosted on CloudFlare" do
setup do
@asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
end
should "return the original file, not the polished file" do
@source = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
assert_downloaded(517_706, @source) # polished size: 502_052
assert_downloaded(517_706, @asset) # polished size: 502_052
end
end
@@ -36,11 +40,12 @@ module Downloads
setup do
@source = "https://dantewontdie.artstation.com/projects/YZK5q"
@download = Downloads::File.new(@source)
@download.download!
end
should "download the original image instead" do
assert_equal("https://cdna.artstation.com/p/assets/images/images/006/066/534/large/yinan-cui-reika.jpg?1495781565", @download.source)
file, strategy = @download.download!
assert_equal(237_651, ::File.size(file.path))
end
end
end

6
test/unit/downloads/deviant_art_test.rb
View File

@@ -8,11 +8,7 @@ module Downloads
@source = "http://starbitt.deviantart.com/art/09271X-636962118"
@download = Downloads::File.new(@source)
@tempfile = @download.download!
end
should "set the html page as the source" do
assert_equal("https://orig00.deviantart.net/82ef/f/2016/271/7/1/aaaaaa_by_starbitt-daj8b46.gif", @download.source)
@tempfile, strategy = @download.download!
end
should "work" do

2
test/unit/downloads/file_test.rb
View File

@@ -41,7 +41,7 @@ module Downloads
end
should "store the file in the tempfile path" do
tempfile = @download.download!
tempfile, strategy = @download.download!
assert_equal(@source, @download.source)
assert_operator(tempfile.size, :>, 0, "should have data")
end

68
test/unit/downloads/pixiv_test.rb
View File

@@ -4,6 +4,7 @@ module Downloads
class PixivTest < ActiveSupport::TestCase
def setup
super
Downloads::File.stubs(:is_cloudflare?).returns(false)
load_pixiv_tokens!
end
@@ -13,29 +14,6 @@ module Downloads
end
context "in all cases" do
# Test an old illustration (one uploaded before 2014-09-16). New
# /img-original/ and /img-master/ URLs currently don't work for images
# uploaded before this date. Only old /imgXX/img/username/ URLs work.
context "downloading an old PNG illustration" do
setup do
@medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=14901720"
@big_page = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=14901720"
@new_small_thumbnail = "http://i1.pixiv.net/c/150x150/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg"
@new_medium_thumbnail = "http://i1.pixiv.net/c/600x600/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg"
@new_full_size_image = "http://i1.pixiv.net/img-original/img/2010/11/30/08/39/58/14901720_p0.png"
@file_size = 1261
end
should "work when using new URLs" do
# Don't know the actual file size of the thumbnails since they don't work.
assert_downloaded(1083, @new_small_thumbnail)
assert_downloaded(1083, @new_medium_thumbnail)
assert_downloaded(@file_size, @new_full_size_image)
end
end
# Test a new illustration (one uploaded after 2014-09-30). New illustrations
# must use /img-original/ for full size URLs. Old /imgXX/img/username/ style URLs
# don't work for images uploaded after this date.
@@ -103,21 +81,6 @@ module Downloads
end
end
context "downloading a bad id image" do
setup do
@bad_id_full = "https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png"
@bad_id_sample = "https://i.pximg.net/c/600x600/img-master/img/2017/11/22/01/06/44/65991677_p0_master1200.jpg"
end
should "not raise an error when rewriting the url" do
assert_nothing_raised { assert_not_rewritten(@bad_id_full) }
end
should_eventually "rewrite bad id samples to full size" do
assert_rewritten(@bad_id_full, @bad_id_sample)
end
end
context "downloading a ugoira" do
setup do
@medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364"
@@ -138,6 +101,8 @@ module Downloads
context "downloading a profile image" do
should "download new profile images" do
skip "profile images are no longer supported"
@file_url = "https://i.pximg.net/user-profile/img/2014/12/18/10/31/23/8733472_7dc7310db6cc37163af145d04499e411_170.jpg"
@file_size = 23_328
@@ -149,8 +114,10 @@ module Downloads
context "downloading a background image" do
should "download the image" do
@file_url = "http://i1.pixiv.net/background/img/2016/05/17/12/05/48/2074388_d4ac52034f7ca0af3e083d59fde7e97f.jpg"
@file_size = 386_678
skip "background images are no longer supported"
@file_url = "https://i.pximg.net/background/img/2015/10/25/08/45/27/198128_77ddf78cdb162e3d1c0d5134af185813.jpg"
@file_size = 0
assert_not_rewritten(@file_url)
assert_downloaded(@file_size, @file_url)
@@ -159,21 +126,23 @@ module Downloads
context "downloading a novel image" do
should "download new novel images" do
@file_url = "http://i1.pixiv.net/novel-cover-original/img/2016/11/03/20/10/58/7436075_f75af69f3eacd1656d3733c72aa959cf.jpg"
@file_size = 316_311
@file_url = "https://i.pximg.net/novel-cover-original/img/2017/07/27/23/14/17/8465454_80685d10e6df4d7d53ad347ddc18a36b.jpg"
@ref = 'https://www.pixiv.net/novel/show.php?id=8465454&mode=cover'
@file_size = 532_129
assert_not_rewritten(@file_url)
assert_downloaded(@file_size, @file_url)
assert_not_rewritten(@file_url, @ref)
assert_downloaded(@file_size, @file_url, @ref)
end
end
context "downloading a pixiv fanbox image" do
should "work" do
@file_url = "https://fanbox.pixiv.net/images/post/31757/w/1200/0CdXtgr4al3t43gQG4NZLnpQ.jpeg"
@file_size = 200_239
@source = "https://www.pixiv.net/fanbox/creator/12491073/post/82406"
@file_url = "https://fanbox.pixiv.net/images/post/82406/D833IKA7FIesJXL8xx39rrG0.jpeg"
@file_size = 873_387
assert_not_rewritten(@file_url)
assert_downloaded(@file_size, @file_url)
assert_not_rewritten(@file_url, @source)
assert_downloaded(@file_size, @file_url, @source)
end
end
end
@@ -181,12 +150,11 @@ module Downloads
context "An ugoira site for pixiv" do
setup do
@download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@tempfile = @download.download!
@tempfile, strategy = @download.download!
@tempfile.close!
end
should "capture the data" do
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @download.source)
assert_equal(2, @download.data[:ugoira_frame_data].size)
if @download.data[:ugoira_frame_data][0]["file"]
assert_equal([{"file"=>"000000.jpg", "delay"=>125}, {"file"=>"000001.jpg", "delay"=>125}], @download.data[:ugoira_frame_data])

80
test/unit/downloads/tumblr_test.rb
View File

@@ -2,43 +2,52 @@ require 'test_helper'
module Downloads
class TumblrTest < ActiveSupport::TestCase
# Currently there's no way to obtain the raw version of these images,
# so we have to change the tests to validate against the 1280 version
context "a download for a tumblr 500 sample" do
should "instead download the raw version" do
should "instead download the 1280 version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@ref = "https://noizave.tumblr.com/post/162206271767"
@source = "https://24.media.tumblr.com/fc328250915434e66e8e6a92773f79d0/tumblr_mf4nshfibc1s0oswoo1_500.jpg"
@rewrite = "http://data.tumblr.com/fc328250915434e66e8e6a92773f79d0/tumblr_mf4nshfibc1s0oswoo1_raw.jpg"
assert_rewritten(@rewrite, @source)
assert_downloaded(196_617, @source)
@rewrite = "https://media.tumblr.com/fc328250915434e66e8e6a92773f79d0/tumblr_mf4nshfibc1s0oswoo1_1280.jpg"
assert_rewritten(@rewrite, @source, @ref)
assert_downloaded(113909, @source, @ref)
# assert_downloaded(196_617, @source)
end
end
context "a download for a *.media.tumblr.com/tumblr_$id_$size image without a larger size" do
should "download the same version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@ref = "https://noizave.tumblr.com/post/162206271767"
@source = "https://25.media.tumblr.com/tumblr_lxbzel2H5y1r9yjhso1_500.jpg"
@rewrite = "http://data.tumblr.com/tumblr_lxbzel2H5y1r9yjhso1_500.jpg"
assert_rewritten(@rewrite, @source)
assert_downloaded(90_122, @source)
@rewrite = "https://media.tumblr.com/tumblr_lxbzel2H5y1r9yjhso1_1280.jpg"
assert_rewritten(@rewrite, @source, @ref)
assert_downloaded(41803, @source, @ref)
# assert_downloaded(90_122, @source)
end
end
context "a download for a *.media.tumblr.com/tumblr_$id_$size image with a larger size" do
should "download the best available version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@ref = "https://noizave.tumblr.com/post/162206271767"
@source = "https://25.media.tumblr.com/tumblr_m2dxb8aOJi1rop2v0o1_500.png"
@rewrite = "http://data.tumblr.com/tumblr_m2dxb8aOJi1rop2v0o1_1280.png"
assert_rewritten(@rewrite, @source)
assert_downloaded(34_060, @source)
@rewrite = "https://media.tumblr.com/tumblr_m2dxb8aOJi1rop2v0o1_1280.png"
assert_rewritten(@rewrite, @source, @ref)
assert_downloaded(62658, @source, @ref)
end
end
context "a download for a *.media.tumblr.com/$hash/tumblr_$id_rN_$size image" do
should "download the best available version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@ref = "https://noizave.tumblr.com/post/162206271767"
@source = "https://33.media.tumblr.com/4b7fecf9a5a8284fbaefb051a2369b55/tumblr_npozqfwc9h1rt6u7do1_r1_500.gif"
@rewrite = "http://data.tumblr.com/4b7fecf9a5a8284fbaefb051a2369b55/tumblr_npozqfwc9h1rt6u7do1_r1_raw.gif"
assert_rewritten(@rewrite, @source)
assert_downloaded(1_234_017, @source)
@rewrite = "https://media.tumblr.com/4b7fecf9a5a8284fbaefb051a2369b55/tumblr_npozqfwc9h1rt6u7do1_r1_1280.gif"
assert_rewritten(@rewrite, @source, @ref)
assert_downloaded(1_234_017, @source, @ref)
end
end
@@ -46,40 +55,33 @@ module Downloads
should "download the best available version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@source = "https://68.media.tumblr.com/ee02048f5578595badc95905e17154b4/tumblr_inline_ofbr4452601sk4jd9_250.gif"
@rewrite = "http://data.tumblr.com/ee02048f5578595badc95905e17154b4/tumblr_inline_ofbr4452601sk4jd9_500.gif"
assert_rewritten(@rewrite, @source)
assert_downloaded(110_348, @source)
@rewrite = "https://media.tumblr.com/ee02048f5578595badc95905e17154b4/tumblr_inline_ofbr4452601sk4jd9_1280.gif"
assert_rewritten(@rewrite, @source, @ref)
assert_downloaded(110_348, @source, @ref)
end
end
context "a download for a data.tumblr.com/$id_$size image with a larger size" do
context "a download for a media.tumblr.com/$id_$size image with a larger size" do
should "download the best available version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@source = "http://data.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_400.jpg"
@rewrite = "http://data.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_500.jpg"
assert_rewritten(@rewrite, @source)
assert_downloaded(153_885, @source)
@ref = "https://noizave.tumblr.com/post/162206271767"
@source = "http://media.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_400.jpg"
@rewrite = "https://media.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_1280.jpg"
assert_rewritten(@rewrite, @source, @ref)
assert_downloaded(122413, @source)
# assert_downloaded(153_885, @source)
end
end
context "a download for a data.tumblr.com/tumblr_$id_$size.jpg image" do
context "a download for a media.tumblr.com/tumblr_$id_$size.jpg image" do
should "download the best available version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@source = "http://data.tumblr.com/tumblr_m24kbxqKAX1rszquso1_250.jpg"
@rewrite = "http://data.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg"
assert_rewritten(@rewrite, @source)
assert_downloaded(296_399, @source)
end
end
context "a download for a gs1.wac.edgecastcdn.net image" do
should "rewrite to the full tumblr version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@source = "https://gs1.wac.edgecastcdn.net/8019B6/data.tumblr.com/tumblr_m2dxb8aOJi1rop2v0o1_500.png"
@rewrite = "http://data.tumblr.com/tumblr_m2dxb8aOJi1rop2v0o1_1280.png"
assert_downloaded(34_060, @source)
assert_rewritten(@rewrite, @source)
@ref = "https://noizave.tumblr.com/post/162206271767"
@source = "http://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_250.jpg"
@rewrite = "https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg"
assert_rewritten(@rewrite, @source, @ref)
assert_downloaded(101869, @source, @ref)
# assert_downloaded(296_399, @source)
end
end
@@ -87,9 +89,9 @@ module Downloads
should "download the best available version" do
skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
@source = "https://noizave.tumblr.com/post/162206271767"
@rewrite = "http://data.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_raw.png"
@rewrite = "https://media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_1280.png"
assert_downloaded(3_620, @source)
assert_downloaded(3655, @source)
assert_rewritten(@rewrite, @source)
end
end

15
test/unit/downloads/twitter_test.rb
View File

@@ -8,17 +8,19 @@ module Downloads
@source = "https://twitter.com/CincinnatiZoo/status/859073537713328129"
@rewrite = "https://video.twimg.com/ext_tw_video/859073467769126913/pu/vid/1280x720/cPGgVROXHy3yrK6u.mp4"
assert_rewritten(@rewrite, @source)
assert_downloaded(8_602_983, @source)
# this takes awhile so just skip it unless we really want to test it
# assert_downloaded(8_602_983, @source)
end
end
context "downloading a 'https://twitter.com/:user/status/:id/photo/:n' card url" do
should "download the orig file" do
skip "Twitter key is not set" unless Danbooru.config.twitter_api_key
@source = "https://twitter.com/masayasuf/status/870734961778630656/photo/1"
@rewrite = "https://pbs.twimg.com/media/DBV40M2UIAAHYlt.jpg:orig"
@source = "https://twitter.com/ry_o_ta_/status/1024316791688843269/photo/1"
@rewrite = "https://pbs.twimg.com/media/Djcar72VsAAZsGa.jpg:orig"
assert_rewritten(@rewrite, @source)
assert_downloaded(788_206, @source)
assert_downloaded(103812, @source)
end
end
@@ -37,8 +39,9 @@ module Downloads
skip "Twitter key is not set" unless Danbooru.config.twitter_api_key
@source = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:large"
@rewrite = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:orig"
assert_rewritten(@rewrite, @source)
assert_downloaded(9800, @source)
@ref = "https://twitter.com/nounproject/status/540944400767922176"
assert_rewritten(@rewrite, @source, @ref)
assert_downloaded(9800, @source, @ref)
end
end
end