Rewrite tests for downloading new Pixiv URLs.
* Test that downloading the HTML page downloads the full size image instead. * Test that downloading a small or medium size image downloads the full size instead. * Test the above for both single illustrations and for manga galleries, for new and old posts (i.e. uploaded before the Pixiv URL changes versus after), and for different file extensions. * Test trying to download ugoira zip files. These tests are expected to fail because we don't support ugoira yet. * Also add some more tests for fetching source data.
This commit is contained in:
evazion
@@ -2,72 +2,270 @@ require 'test_helper'
|
||||
|
||||
module Downloads
|
||||
class PixivTest < ActiveSupport::TestCase
|
||||
context "a download for a pixiv manga page" do
|
||||
setup do
|
||||
@source = "http://img65.pixiv.net/img/kiyoringo/21755794_p2.png"
|
||||
@tempfile = Tempfile.new("danbooru-test")
|
||||
@download = Downloads::File.new(@source, @tempfile.path)
|
||||
VCR.use_cassette("download-pixiv-manga", :record => :new_episodes) do
|
||||
@download.download!
|
||||
def assert_downloaded(expected_filesize, source, cassette)
|
||||
tempfile = Tempfile.new("danbooru-test")
|
||||
download = Downloads::File.new(source, tempfile.path)
|
||||
|
||||
VCR.use_cassette(cassette, :record => :once) do
|
||||
assert_nothing_raised(Downloads::File::Error) do
|
||||
download.download!
|
||||
end
|
||||
end
|
||||
|
||||
should "instead download the big version" do
|
||||
assert_equal("http://img65.pixiv.net/img/kiyoringo/21755794_big_p2.png", @download.source)
|
||||
assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
|
||||
end
|
||||
|
||||
def assert_rewritten(expected_source, test_source)
|
||||
tempfile = Tempfile.new("danbooru-test")
|
||||
download = Downloads::File.new(test_source, tempfile.path)
|
||||
|
||||
rewritten_source, headers = download.before_download(test_source, {})
|
||||
assert_equal(expected_source, rewritten_source, "Tested source URL: #{test_source}")
|
||||
end
|
||||
|
||||
def assert_not_rewritten(source)
|
||||
assert_rewritten(source, source)
|
||||
end
|
||||
|
||||
|
||||
|
||||
# Test an old illustration (one uploaded before 2014-09-16). New
|
||||
# /img-original/ and /img-master/ URLs currently don't work for images
|
||||
# uploaded before this date. Only old /imgXX/img/username/ URLs work.
|
||||
context "downloading an old PNG illustration" do
|
||||
setup do
|
||||
@medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=14901720"
|
||||
@big_page = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=14901720"
|
||||
|
||||
@old_small_thumbnail = "http://i2.pixiv.net/img18/img/evazion/14901720_s.png"
|
||||
@old_medium_thumbnail = "http://i2.pixiv.net/img18/img/evazion/14901720_m.png"
|
||||
@old_full_size_image = "http://i2.pixiv.net/img18/img/evazion/14901720.png"
|
||||
|
||||
@new_small_thumbnail = "http://i1.pixiv.net/c/150x150/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg"
|
||||
@new_medium_thumbnail = "http://i1.pixiv.net/c/600x600/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg"
|
||||
@new_full_size_image = "http://i1.pixiv.net/img-original/img/2010/11/30/08/39/58/14901720_p0.png"
|
||||
|
||||
@file_size = 1_083
|
||||
end
|
||||
|
||||
should "download the full size image" do
|
||||
assert_not_rewritten(@full_size_image)
|
||||
assert_downloaded(@file_size, @old_full_size_image, "download-pixiv-old-png-full-size")
|
||||
end
|
||||
|
||||
should "download the full size image instead of the HTML page" do
|
||||
VCR.use_cassette("rewrite-pixiv-old-png-html-pages", :record => :once) do
|
||||
assert_rewritten(@old_full_size_image, @medium_page)
|
||||
assert_rewritten(@old_full_size_image, @big_page)
|
||||
end
|
||||
|
||||
assert_downloaded(@file_size, @medium_page, "download-pixiv-old-png-medium-html")
|
||||
assert_downloaded(@file_size, @big_page, "download-pixiv-old-png-big-html")
|
||||
end
|
||||
|
||||
should "download the full size image instead of the thumbnail" do
|
||||
VCR.use_cassette("rewrite-pixiv-old-png-thumbnails", :record => :once) do
|
||||
assert_rewritten(@old_full_size_image, @old_small_thumbnail)
|
||||
assert_rewritten(@old_full_size_image, @old_medium_thumbnail)
|
||||
end
|
||||
|
||||
assert_downloaded(@file_size, @old_small_thumbnail, "download-pixiv-old-png-old-small-thumbnail")
|
||||
assert_downloaded(@file_size, @old_medium_thumbnail, "download-pixiv-old-png-old-medium-thumbnail")
|
||||
end
|
||||
|
||||
# These tests aren't expected to pass. This is just to test if Pixiv has changed to using new URLs for old images.
|
||||
should_eventually "work when using new URLs" do
|
||||
# Don't know the actual file size of the thumbnails since they don't work.
|
||||
assert_downloaded(42, @new_small_thumbnail, "download-pixiv-old-png-new-small-thumbnail")
|
||||
assert_downloaded(42, @new_medium_thumbnail, "download-pixiv-old-png-new-medium-thumbnail")
|
||||
|
||||
assert_downloaded(@file_size, @new_full_size_image, "download-pixiv-old-png-new-full-size")
|
||||
end
|
||||
end
|
||||
|
||||
context "a download for a html page" do
|
||||
|
||||
|
||||
# Test a new illustration (one uploaded after 2014-09-30). New illustrations
|
||||
# must use /img-original/ for full size URLs. Old /imgXX/img/username/ style URLs
|
||||
# don't work for images uploaded after this date.
|
||||
context "downloading a new PNG illustration" do
|
||||
setup do
|
||||
# @source = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=23828655"
|
||||
@source = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=4348318"
|
||||
@tempfile = Tempfile.new("danbooru-test")
|
||||
@download = Downloads::File.new(@source, @tempfile.path)
|
||||
VCR.use_cassette("download-pixiv-html", :record => :new_episodes) do
|
||||
@download.download!
|
||||
end
|
||||
@medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46337015"
|
||||
@big_page = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=46337015"
|
||||
|
||||
@medium_thumbnail = "http://i2.pixiv.net/c/600x600/img-master/img/2014/10/04/03/59/52/46337015_p0_master1200.jpg"
|
||||
@full_size_image = "http://i2.pixiv.net/img-original/img/2014/10/04/03/59/52/46337015_p0.png"
|
||||
|
||||
@file_size = 5_141
|
||||
end
|
||||
|
||||
should "work" do
|
||||
assert_equal(185_778, ::File.size(@tempfile.path))
|
||||
should "download the full size image" do
|
||||
assert_not_rewritten(@full_size_image)
|
||||
|
||||
assert_downloaded(@file_size, @full_size_image, "download-pixiv-new-png-full-size")
|
||||
end
|
||||
|
||||
should "download the full size image instead of the HTML page" do
|
||||
VCR.use_cassette("rewrite-pixiv-new-png-html-pages", :record => :once) do
|
||||
assert_rewritten(@full_size_image, @medium_page)
|
||||
assert_rewritten(@full_size_image, @big_page)
|
||||
end
|
||||
|
||||
assert_downloaded(@file_size, @medium_page, "download-pixiv-new-png-medium-html")
|
||||
assert_downloaded(@file_size, @big_page, "download-pixiv-new-png-big-html")
|
||||
end
|
||||
|
||||
should "download the full size image instead of the thumbnail" do
|
||||
VCR.use_cassette("rewrite-pixiv-new-png-thumbnails", :record => :once) do
|
||||
assert_rewritten(@full_size_image, @medium_thumbnail)
|
||||
end
|
||||
|
||||
assert_downloaded(@file_size, @medium_thumbnail, "download-pixiv-new-png-medium-thumbnail")
|
||||
end
|
||||
end
|
||||
|
||||
context "a download for a small image thumbnail" do
|
||||
|
||||
|
||||
context "downloading an old manga image" do
|
||||
setup do
|
||||
@source = "http://img02.pixiv.net/img/wanwandoh/4348318_s.jpg"
|
||||
@tempfile = Tempfile.new("danbooru-test")
|
||||
@download = Downloads::File.new(@source, @tempfile.path)
|
||||
VCR.use_cassette("download-pixiv-small", :record => :new_episodes) do
|
||||
@download.download!
|
||||
@medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054"
|
||||
@manga_page = "http://www.pixiv.net/member_illust.php?mode=manga&illust_id=18557054"
|
||||
@manga_big_p1_page = "http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=18557054&page=1"
|
||||
|
||||
@p0_tiny_thumbnail = "http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg"
|
||||
@p0_small_thumbnail = "http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_s.png"
|
||||
@p0_medium_thumbnail = "http://i1.pixiv.net/img07/img/pasirism/18557054_m.png"
|
||||
@p0_large_thumbnail = "http://i1.pixiv.net/img07/img/pasirism/18557054_p0.png"
|
||||
@p1_large_thumbnail = "http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png"
|
||||
|
||||
@p0_full_size_image = "http://i1.pixiv.net/img07/img/pasirism/18557054_big_p0.png"
|
||||
@p1_full_size_image = "http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png"
|
||||
|
||||
@p0_file_size = 1964
|
||||
@p1_file_size = 1927
|
||||
end
|
||||
|
||||
should "download the full size image" do
|
||||
assert_not_rewritten(@p0_full_size_image)
|
||||
assert_not_rewritten(@p1_full_size_image)
|
||||
|
||||
assert_downloaded(@p0_file_size, @p0_full_size_image, "download-pixiv-old-manga-p0-full-size")
|
||||
assert_downloaded(@p1_file_size, @p1_full_size_image, "download-pixiv-old-manga-p1-full-size")
|
||||
end
|
||||
|
||||
should "download the full size image instead of the HTML page" do
|
||||
VCR.use_cassette("rewrite-pixiv-old-manga-html-pages", :record => :once) do
|
||||
assert_rewritten(@p0_full_size_image, @medium_page)
|
||||
assert_rewritten(@p0_full_size_image, @manga_page)
|
||||
assert_rewritten(@p1_full_size_image, @manga_big_p1_page)
|
||||
end
|
||||
|
||||
assert_downloaded(@p0_file_size, @medium_page, "download-pixiv-old-manga-p0-medium-html")
|
||||
assert_downloaded(@p0_file_size, @manga_page, "download-pixiv-old-manga-p0-big-html")
|
||||
assert_downloaded(@p1_file_size, @manga_big_p1_page, "download-pixiv-old-manga-p1-big-html")
|
||||
end
|
||||
|
||||
should "instead download the original version" do
|
||||
assert_equal("http://img02.pixiv.net/img/wanwandoh/4348318.jpg", @download.source)
|
||||
should "download the full size image instead of the thumbnail" do
|
||||
VCR.use_cassette("rewrite-pixiv-old-manga-thumbnails", :record => :once) do
|
||||
assert_rewritten(@p0_full_size_image, @p0_tiny_thumbnail)
|
||||
assert_rewritten(@p0_full_size_image, @p0_small_thumbnail)
|
||||
assert_rewritten(@p0_full_size_image, @p0_large_thumbnail)
|
||||
assert_rewritten(@p1_full_size_image, @p1_large_thumbnail)
|
||||
end
|
||||
|
||||
assert_downloaded(@p0_file_size, @p0_tiny_thumbnail, "download-pixiv-old-manga-p0-tiny-thumbnail")
|
||||
assert_downloaded(@p0_file_size, @p0_small_thumbnail, "download-pixiv-old-manga-p0-small-thumbnail")
|
||||
assert_downloaded(@p0_file_size, @p0_large_thumbnail, "download-pixiv-old-manga-p0-large-thumbnail")
|
||||
assert_downloaded(@p1_file_size, @p1_large_thumbnail, "download-pixiv-old-manga-p1-large-thumbnail")
|
||||
end
|
||||
|
||||
should "work" do
|
||||
assert_equal(185_778, ::File.size(@tempfile.path))
|
||||
should "download the full size image instead of the medium thumbnail" do
|
||||
VCR.use_cassette("rewrite-pixiv-old-manga-medium-thumbnail", :record => :once) do
|
||||
assert_rewritten(@p0_full_size_image, @p0_medium_thumbnail)
|
||||
end
|
||||
|
||||
assert_downloaded(@p0_file_size, @p0_medium_thumbnail, "download-pixiv-old-manga-p0-medium-thumbnail")
|
||||
end
|
||||
end
|
||||
|
||||
context "a download for a medium image thumbnail" do
|
||||
|
||||
|
||||
context "downloading a new manga image" do
|
||||
setup do
|
||||
@source = "http://img02.pixiv.net/img/wanwandoh/4348318_m.jpg"
|
||||
@tempfile = Tempfile.new("danbooru-test")
|
||||
@download = Downloads::File.new(@source, @tempfile.path)
|
||||
VCR.use_cassette("download-pixiv-medium", :record => :new_episodes) do
|
||||
@download.download!
|
||||
@medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46304614"
|
||||
@manga_page = "http://www.pixiv.net/member_illust.php?mode=manga&illust_id=46304614"
|
||||
@manga_big_p1_page = "http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=46304614&page=1"
|
||||
|
||||
@p0_large_thumbnail = "http://i1.pixiv.net/c/1200x1200/img-master/img/2014/10/02/14/21/39/46304614_p0_master1200.jpg"
|
||||
@p1_large_thumbnail = "http://i1.pixiv.net/c/1200x1200/img-master/img/2014/10/02/14/21/39/46304614_p1_master1200.jpg"
|
||||
@p0_full_size_image = "http://i1.pixiv.net/img-original/img/2014/10/02/14/21/39/46304614_p0.gif"
|
||||
@p1_full_size_image = "http://i1.pixiv.net/img-original/img/2014/10/02/14/21/39/46304614_p1.gif"
|
||||
|
||||
@p0_file_size = 61_131
|
||||
@p1_file_size = 46_818
|
||||
end
|
||||
|
||||
should "download the full size image" do
|
||||
assert_not_rewritten(@p0_full_size_image)
|
||||
assert_not_rewritten(@p1_full_size_image)
|
||||
|
||||
assert_downloaded(@p0_file_size, @p0_full_size_image, "download-pixiv-new-manga-p0-full-size")
|
||||
assert_downloaded(@p1_file_size, @p1_full_size_image, "download-pixiv-new-manga-p1-full-size")
|
||||
end
|
||||
|
||||
should "download the full size image instead of the HTML page" do
|
||||
VCR.use_cassette("rewrite-pixiv-new-manga-html-pages", :record => :once) do
|
||||
assert_rewritten(@p0_full_size_image, @medium_page)
|
||||
assert_rewritten(@p0_full_size_image, @manga_page)
|
||||
assert_rewritten(@p1_full_size_image, @manga_big_p1_page)
|
||||
end
|
||||
|
||||
assert_downloaded(@p0_file_size, @medium_page, "download-pixiv-new-manga-p0-medium-html")
|
||||
assert_downloaded(@p0_file_size, @manga_page, "download-pixiv-new-manga-p0-big-html")
|
||||
assert_downloaded(@p1_file_size, @manga_big_p1_page, "download-pixiv-new-manga-p1-big-html")
|
||||
end
|
||||
|
||||
should "instead download the original version" do
|
||||
assert_equal("http://img02.pixiv.net/img/wanwandoh/4348318.jpg", @download.source)
|
||||
should "download the full size image instead of the thumbnail" do
|
||||
VCR.use_cassette("rewrite-pixiv-new-manga-thumbnails", :record => :once) do
|
||||
assert_rewritten(@p0_full_size_image, @p0_large_thumbnail)
|
||||
assert_rewritten(@p1_full_size_image, @p1_large_thumbnail)
|
||||
end
|
||||
|
||||
assert_downloaded(@p0_file_size, @p0_large_thumbnail, "download-pixiv-new-manga-p0-large-thumbnail")
|
||||
assert_downloaded(@p1_file_size, @p1_large_thumbnail, "download-pixiv-new-manga-p1-large-thumbnail")
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
context "downloading a ugoira" do
|
||||
setup do
|
||||
@medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46323924"
|
||||
@small_thumbnail = "http://i1.pixiv.net/img-inf/img/2014/10/03/17/29/16/46323924_s.jpg"
|
||||
@zip_file = "http://i1.pixiv.net/img-zip-ugoira/img/2014/10/03/17/29/16/46323924_ugoira1920x1080.zip"
|
||||
@file_size = 41_171
|
||||
end
|
||||
|
||||
should "work" do
|
||||
assert_equal(185_778, ::File.size(@tempfile.path))
|
||||
should "download the zip file instead of the HTML page" do
|
||||
VCR.use_cassette("rewrite-pixiv-ugoira-html-pages", :record => :once) do
|
||||
assert_rewritten(@zip_file, @medium_page)
|
||||
end
|
||||
|
||||
assert_downloaded(@file_size, @medium_page, "download-pixiv-ugoira-medium-page")
|
||||
end
|
||||
|
||||
should "download the zip file instead of the thumbnail" do
|
||||
VCR.use_cassette("rewrite-pixiv-ugoira-thumbnails", :record => :once) do
|
||||
assert_rewritten(@zip_file, @small_thumbnail)
|
||||
end
|
||||
|
||||
assert_downloaded(@file_size, @small_thumbnail, "download-pixiv-ugoira-small-thumbnail")
|
||||
end
|
||||
|
||||
should "download the zip file" do
|
||||
assert_not_rewritten(@zip_file)
|
||||
|
||||
assert_downloaded(@file_size, @zip_file, "download-pixiv-ugoira-zip-file")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -4,32 +4,41 @@ require 'test_helper'
|
||||
|
||||
module Sources
|
||||
class PixivTest < ActiveSupport::TestCase
|
||||
context "The source site for pixiv" do
|
||||
def get_source(source, cassette)
|
||||
VCR.use_cassette(cassette, :record => :once) do
|
||||
@site = Sources::Site.new(source)
|
||||
@site.get
|
||||
@site
|
||||
end
|
||||
end
|
||||
|
||||
context "fetching source data for a new manga image" do
|
||||
setup do
|
||||
VCR.use_cassette("source-pixiv-unit-test", :record => :new_episodes) do
|
||||
@site = Sources::Site.new("http://www.pixiv.net/member_illust.php?mode=big&illust_id=9646484")
|
||||
@site.get
|
||||
end
|
||||
get_source("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46324488", "source-pixiv-new-manga")
|
||||
end
|
||||
|
||||
should "get the profile" do
|
||||
assert_equal("http://www.pixiv.net/member.php?id=4015", @site.profile_url)
|
||||
assert_equal("http://www.pixiv.net/member.php?id=339253", @site.profile_url)
|
||||
end
|
||||
|
||||
should "get the artist name" do
|
||||
assert_equal("シビレ罠", @site.artist_name)
|
||||
assert_equal("evazion", @site.artist_name)
|
||||
end
|
||||
|
||||
should "get the image url" do
|
||||
assert_equal("http://i1.pixiv.net/img01/img/nisieda/9646484.jpg", @site.image_url)
|
||||
should "get the full size image url" do
|
||||
assert_equal("http://i1.pixiv.net/img-original/img/2014/10/03/18/10/20/46324488_p0.png", @site.image_url)
|
||||
end
|
||||
|
||||
should "get the page count" do
|
||||
assert_equal(3, @site.page_count)
|
||||
end
|
||||
|
||||
should "get the tags" do
|
||||
assert(@site.tags.size > 0)
|
||||
first_tag = @site.tags.first
|
||||
assert_equal(2, first_tag.size)
|
||||
assert(first_tag[0] =~ /./)
|
||||
assert(first_tag[1] =~ /search\.php/)
|
||||
pixiv_tags = @site.tags.map(&:first)
|
||||
pixiv_links = @site.tags.map(&:last)
|
||||
|
||||
assert_equal(%w(R-18G derp tag1 tag2 オリジナル), pixiv_tags)
|
||||
assert_contains(pixiv_links, /search\.php/)
|
||||
end
|
||||
|
||||
should "convert a page into a json representation" do
|
||||
@@ -38,5 +47,55 @@ module Sources
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context "fetching source data for an old manga image" do
|
||||
setup do
|
||||
get_source("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=45792845", "source-pixiv-old-manga")
|
||||
end
|
||||
|
||||
should "get the page count" do
|
||||
assert_equal(3, @site.page_count)
|
||||
end
|
||||
|
||||
should "get the full size image url" do
|
||||
assert_equal("http://i2.pixiv.net/img18/img/ringo78/45792845_big_p0.jpg", @site.image_url)
|
||||
end
|
||||
end
|
||||
|
||||
context "fetching source data for a new illustration" do
|
||||
setup do
|
||||
get_source("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46337015", "source-pixiv-new-illust")
|
||||
end
|
||||
|
||||
should "get the page count" do
|
||||
assert_equal(1, @site.page_count)
|
||||
end
|
||||
|
||||
should "get the full size image url" do
|
||||
assert_equal("http://i2.pixiv.net/img-original/img/2014/10/04/03/59/52/46337015_p0.png", @site.image_url)
|
||||
end
|
||||
end
|
||||
|
||||
context "fetching source data for an old illustration" do
|
||||
setup do
|
||||
get_source("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=14901720", "source-pixiv-old-illust")
|
||||
end
|
||||
|
||||
should "get the page count" do
|
||||
assert_equal(1, @site.page_count)
|
||||
end
|
||||
|
||||
should "get the full size image url" do
|
||||
assert_equal("http://i2.pixiv.net/img18/img/evazion/14901720.png", @site.image_url)
|
||||
end
|
||||
|
||||
should "get the tags" do
|
||||
pixiv_tags = @site.tags.map(&:first)
|
||||
pixiv_links = @site.tags.map(&:last)
|
||||
|
||||
assert_equal(%w(derp), pixiv_tags)
|
||||
assert_contains(pixiv_links, /search\.php/)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user