From 339e88584d6816632bc968613006c4533dbdb956 Mon Sep 17 00:00:00 2001 From: evazion Date: Sat, 27 Sep 2014 00:48:07 -0500 Subject: [PATCH] Rewrite tests for downloading new Pixiv URLs. * Test that downloading the HTML page downloads the full size image instead. * Test that downloading a small or medium size image downloads the full size instead. * Test the above for both single illustrations and for manga galleries, for new and old posts (i.e. uploaded before the Pixiv URL changes versus after), and for different file extensions. * Test trying to download ugoira zip files. These tests are expected to fail because we don't support ugoira yet. * Also add some more tests for fetching source data. --- test/unit/downloads/pixiv_test.rb | 276 +++++++++++++++++++++++++----- test/unit/sources/pixiv_test.rb | 87 ++++++++-- 2 files changed, 310 insertions(+), 53 deletions(-) diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb index f28448c7a..868ca7332 100644 --- a/test/unit/downloads/pixiv_test.rb +++ b/test/unit/downloads/pixiv_test.rb @@ -2,72 +2,270 @@ require 'test_helper' module Downloads class PixivTest < ActiveSupport::TestCase - context "a download for a pixiv manga page" do - setup do - @source = "http://img65.pixiv.net/img/kiyoringo/21755794_p2.png" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - VCR.use_cassette("download-pixiv-manga", :record => :new_episodes) do - @download.download! + def assert_downloaded(expected_filesize, source, cassette) + tempfile = Tempfile.new("danbooru-test") + download = Downloads::File.new(source, tempfile.path) + + VCR.use_cassette(cassette, :record => :once) do + assert_nothing_raised(Downloads::File::Error) do + download.download! end end - should "instead download the big version" do - assert_equal("http://img65.pixiv.net/img/kiyoringo/21755794_big_p2.png", @download.source) + assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}") + end + + def assert_rewritten(expected_source, test_source) + tempfile = Tempfile.new("danbooru-test") + download = Downloads::File.new(test_source, tempfile.path) + + rewritten_source, headers = download.before_download(test_source, {}) + assert_equal(expected_source, rewritten_source, "Tested source URL: #{test_source}") + end + + def assert_not_rewritten(source) + assert_rewritten(source, source) + end + + + + # Test an old illustration (one uploaded before 2014-09-16). New + # /img-original/ and /img-master/ URLs currently don't work for images + # uploaded before this date. Only old /imgXX/img/username/ URLs work. + context "downloading an old PNG illustration" do + setup do + @medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=14901720" + @big_page = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=14901720" + + @old_small_thumbnail = "http://i2.pixiv.net/img18/img/evazion/14901720_s.png" + @old_medium_thumbnail = "http://i2.pixiv.net/img18/img/evazion/14901720_m.png" + @old_full_size_image = "http://i2.pixiv.net/img18/img/evazion/14901720.png" + + @new_small_thumbnail = "http://i1.pixiv.net/c/150x150/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg" + @new_medium_thumbnail = "http://i1.pixiv.net/c/600x600/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg" + @new_full_size_image = "http://i1.pixiv.net/img-original/img/2010/11/30/08/39/58/14901720_p0.png" + + @file_size = 1_083 + end + + should "download the full size image" do + assert_not_rewritten(@full_size_image) + assert_downloaded(@file_size, @old_full_size_image, "download-pixiv-old-png-full-size") + end + + should "download the full size image instead of the HTML page" do + VCR.use_cassette("rewrite-pixiv-old-png-html-pages", :record => :once) do + assert_rewritten(@old_full_size_image, @medium_page) + assert_rewritten(@old_full_size_image, @big_page) + end + + assert_downloaded(@file_size, @medium_page, "download-pixiv-old-png-medium-html") + assert_downloaded(@file_size, @big_page, "download-pixiv-old-png-big-html") + end + + should "download the full size image instead of the thumbnail" do + VCR.use_cassette("rewrite-pixiv-old-png-thumbnails", :record => :once) do + assert_rewritten(@old_full_size_image, @old_small_thumbnail) + assert_rewritten(@old_full_size_image, @old_medium_thumbnail) + end + + assert_downloaded(@file_size, @old_small_thumbnail, "download-pixiv-old-png-old-small-thumbnail") + assert_downloaded(@file_size, @old_medium_thumbnail, "download-pixiv-old-png-old-medium-thumbnail") + end + + # These tests aren't expected to pass. This is just to test if Pixiv has changed to using new URLs for old images. + should_eventually "work when using new URLs" do + # Don't know the actual file size of the thumbnails since they don't work. + assert_downloaded(42, @new_small_thumbnail, "download-pixiv-old-png-new-small-thumbnail") + assert_downloaded(42, @new_medium_thumbnail, "download-pixiv-old-png-new-medium-thumbnail") + + assert_downloaded(@file_size, @new_full_size_image, "download-pixiv-old-png-new-full-size") end end - context "a download for a html page" do + + + # Test a new illustration (one uploaded after 2014-09-30). New illustrations + # must use /img-original/ for full size URLs. Old /imgXX/img/username/ style URLs + # don't work for images uploaded after this date. + context "downloading a new PNG illustration" do setup do - # @source = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=23828655" - @source = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=4348318" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - VCR.use_cassette("download-pixiv-html", :record => :new_episodes) do - @download.download! - end + @medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46337015" + @big_page = "http://www.pixiv.net/member_illust.php?mode=big&illust_id=46337015" + + @medium_thumbnail = "http://i2.pixiv.net/c/600x600/img-master/img/2014/10/04/03/59/52/46337015_p0_master1200.jpg" + @full_size_image = "http://i2.pixiv.net/img-original/img/2014/10/04/03/59/52/46337015_p0.png" + + @file_size = 5_141 end - should "work" do - assert_equal(185_778, ::File.size(@tempfile.path)) + should "download the full size image" do + assert_not_rewritten(@full_size_image) + + assert_downloaded(@file_size, @full_size_image, "download-pixiv-new-png-full-size") + end + + should "download the full size image instead of the HTML page" do + VCR.use_cassette("rewrite-pixiv-new-png-html-pages", :record => :once) do + assert_rewritten(@full_size_image, @medium_page) + assert_rewritten(@full_size_image, @big_page) + end + + assert_downloaded(@file_size, @medium_page, "download-pixiv-new-png-medium-html") + assert_downloaded(@file_size, @big_page, "download-pixiv-new-png-big-html") + end + + should "download the full size image instead of the thumbnail" do + VCR.use_cassette("rewrite-pixiv-new-png-thumbnails", :record => :once) do + assert_rewritten(@full_size_image, @medium_thumbnail) + end + + assert_downloaded(@file_size, @medium_thumbnail, "download-pixiv-new-png-medium-thumbnail") end end - context "a download for a small image thumbnail" do + + + context "downloading an old manga image" do setup do - @source = "http://img02.pixiv.net/img/wanwandoh/4348318_s.jpg" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - VCR.use_cassette("download-pixiv-small", :record => :new_episodes) do - @download.download! + @medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054" + @manga_page = "http://www.pixiv.net/member_illust.php?mode=manga&illust_id=18557054" + @manga_big_p1_page = "http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=18557054&page=1" + + @p0_tiny_thumbnail = "http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg" + @p0_small_thumbnail = "http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_s.png" + @p0_medium_thumbnail = "http://i1.pixiv.net/img07/img/pasirism/18557054_m.png" + @p0_large_thumbnail = "http://i1.pixiv.net/img07/img/pasirism/18557054_p0.png" + @p1_large_thumbnail = "http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png" + + @p0_full_size_image = "http://i1.pixiv.net/img07/img/pasirism/18557054_big_p0.png" + @p1_full_size_image = "http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png" + + @p0_file_size = 1964 + @p1_file_size = 1927 + end + + should "download the full size image" do + assert_not_rewritten(@p0_full_size_image) + assert_not_rewritten(@p1_full_size_image) + + assert_downloaded(@p0_file_size, @p0_full_size_image, "download-pixiv-old-manga-p0-full-size") + assert_downloaded(@p1_file_size, @p1_full_size_image, "download-pixiv-old-manga-p1-full-size") + end + + should "download the full size image instead of the HTML page" do + VCR.use_cassette("rewrite-pixiv-old-manga-html-pages", :record => :once) do + assert_rewritten(@p0_full_size_image, @medium_page) + assert_rewritten(@p0_full_size_image, @manga_page) + assert_rewritten(@p1_full_size_image, @manga_big_p1_page) end + + assert_downloaded(@p0_file_size, @medium_page, "download-pixiv-old-manga-p0-medium-html") + assert_downloaded(@p0_file_size, @manga_page, "download-pixiv-old-manga-p0-big-html") + assert_downloaded(@p1_file_size, @manga_big_p1_page, "download-pixiv-old-manga-p1-big-html") end - should "instead download the original version" do - assert_equal("http://img02.pixiv.net/img/wanwandoh/4348318.jpg", @download.source) + should "download the full size image instead of the thumbnail" do + VCR.use_cassette("rewrite-pixiv-old-manga-thumbnails", :record => :once) do + assert_rewritten(@p0_full_size_image, @p0_tiny_thumbnail) + assert_rewritten(@p0_full_size_image, @p0_small_thumbnail) + assert_rewritten(@p0_full_size_image, @p0_large_thumbnail) + assert_rewritten(@p1_full_size_image, @p1_large_thumbnail) + end + + assert_downloaded(@p0_file_size, @p0_tiny_thumbnail, "download-pixiv-old-manga-p0-tiny-thumbnail") + assert_downloaded(@p0_file_size, @p0_small_thumbnail, "download-pixiv-old-manga-p0-small-thumbnail") + assert_downloaded(@p0_file_size, @p0_large_thumbnail, "download-pixiv-old-manga-p0-large-thumbnail") + assert_downloaded(@p1_file_size, @p1_large_thumbnail, "download-pixiv-old-manga-p1-large-thumbnail") end - should "work" do - assert_equal(185_778, ::File.size(@tempfile.path)) + should "download the full size image instead of the medium thumbnail" do + VCR.use_cassette("rewrite-pixiv-old-manga-medium-thumbnail", :record => :once) do + assert_rewritten(@p0_full_size_image, @p0_medium_thumbnail) + end + + assert_downloaded(@p0_file_size, @p0_medium_thumbnail, "download-pixiv-old-manga-p0-medium-thumbnail") end end - context "a download for a medium image thumbnail" do + + + context "downloading a new manga image" do setup do - @source = "http://img02.pixiv.net/img/wanwandoh/4348318_m.jpg" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - VCR.use_cassette("download-pixiv-medium", :record => :new_episodes) do - @download.download! + @medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46304614" + @manga_page = "http://www.pixiv.net/member_illust.php?mode=manga&illust_id=46304614" + @manga_big_p1_page = "http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=46304614&page=1" + + @p0_large_thumbnail = "http://i1.pixiv.net/c/1200x1200/img-master/img/2014/10/02/14/21/39/46304614_p0_master1200.jpg" + @p1_large_thumbnail = "http://i1.pixiv.net/c/1200x1200/img-master/img/2014/10/02/14/21/39/46304614_p1_master1200.jpg" + @p0_full_size_image = "http://i1.pixiv.net/img-original/img/2014/10/02/14/21/39/46304614_p0.gif" + @p1_full_size_image = "http://i1.pixiv.net/img-original/img/2014/10/02/14/21/39/46304614_p1.gif" + + @p0_file_size = 61_131 + @p1_file_size = 46_818 + end + + should "download the full size image" do + assert_not_rewritten(@p0_full_size_image) + assert_not_rewritten(@p1_full_size_image) + + assert_downloaded(@p0_file_size, @p0_full_size_image, "download-pixiv-new-manga-p0-full-size") + assert_downloaded(@p1_file_size, @p1_full_size_image, "download-pixiv-new-manga-p1-full-size") + end + + should "download the full size image instead of the HTML page" do + VCR.use_cassette("rewrite-pixiv-new-manga-html-pages", :record => :once) do + assert_rewritten(@p0_full_size_image, @medium_page) + assert_rewritten(@p0_full_size_image, @manga_page) + assert_rewritten(@p1_full_size_image, @manga_big_p1_page) end + + assert_downloaded(@p0_file_size, @medium_page, "download-pixiv-new-manga-p0-medium-html") + assert_downloaded(@p0_file_size, @manga_page, "download-pixiv-new-manga-p0-big-html") + assert_downloaded(@p1_file_size, @manga_big_p1_page, "download-pixiv-new-manga-p1-big-html") end - should "instead download the original version" do - assert_equal("http://img02.pixiv.net/img/wanwandoh/4348318.jpg", @download.source) + should "download the full size image instead of the thumbnail" do + VCR.use_cassette("rewrite-pixiv-new-manga-thumbnails", :record => :once) do + assert_rewritten(@p0_full_size_image, @p0_large_thumbnail) + assert_rewritten(@p1_full_size_image, @p1_large_thumbnail) + end + + assert_downloaded(@p0_file_size, @p0_large_thumbnail, "download-pixiv-new-manga-p0-large-thumbnail") + assert_downloaded(@p1_file_size, @p1_large_thumbnail, "download-pixiv-new-manga-p1-large-thumbnail") + end + end + + + + context "downloading a ugoira" do + setup do + @medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46323924" + @small_thumbnail = "http://i1.pixiv.net/img-inf/img/2014/10/03/17/29/16/46323924_s.jpg" + @zip_file = "http://i1.pixiv.net/img-zip-ugoira/img/2014/10/03/17/29/16/46323924_ugoira1920x1080.zip" + @file_size = 41_171 end - should "work" do - assert_equal(185_778, ::File.size(@tempfile.path)) + should "download the zip file instead of the HTML page" do + VCR.use_cassette("rewrite-pixiv-ugoira-html-pages", :record => :once) do + assert_rewritten(@zip_file, @medium_page) + end + + assert_downloaded(@file_size, @medium_page, "download-pixiv-ugoira-medium-page") + end + + should "download the zip file instead of the thumbnail" do + VCR.use_cassette("rewrite-pixiv-ugoira-thumbnails", :record => :once) do + assert_rewritten(@zip_file, @small_thumbnail) + end + + assert_downloaded(@file_size, @small_thumbnail, "download-pixiv-ugoira-small-thumbnail") + end + + should "download the zip file" do + assert_not_rewritten(@zip_file) + + assert_downloaded(@file_size, @zip_file, "download-pixiv-ugoira-zip-file") end end end diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index 9dc3d9524..1f4bc7938 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -4,32 +4,41 @@ require 'test_helper' module Sources class PixivTest < ActiveSupport::TestCase - context "The source site for pixiv" do + def get_source(source, cassette) + VCR.use_cassette(cassette, :record => :once) do + @site = Sources::Site.new(source) + @site.get + @site + end + end + + context "fetching source data for a new manga image" do setup do - VCR.use_cassette("source-pixiv-unit-test", :record => :new_episodes) do - @site = Sources::Site.new("http://www.pixiv.net/member_illust.php?mode=big&illust_id=9646484") - @site.get - end + get_source("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46324488", "source-pixiv-new-manga") end should "get the profile" do - assert_equal("http://www.pixiv.net/member.php?id=4015", @site.profile_url) + assert_equal("http://www.pixiv.net/member.php?id=339253", @site.profile_url) end should "get the artist name" do - assert_equal("シビレ罠", @site.artist_name) + assert_equal("evazion", @site.artist_name) end - should "get the image url" do - assert_equal("http://i1.pixiv.net/img01/img/nisieda/9646484.jpg", @site.image_url) + should "get the full size image url" do + assert_equal("http://i1.pixiv.net/img-original/img/2014/10/03/18/10/20/46324488_p0.png", @site.image_url) + end + + should "get the page count" do + assert_equal(3, @site.page_count) end should "get the tags" do - assert(@site.tags.size > 0) - first_tag = @site.tags.first - assert_equal(2, first_tag.size) - assert(first_tag[0] =~ /./) - assert(first_tag[1] =~ /search\.php/) + pixiv_tags = @site.tags.map(&:first) + pixiv_links = @site.tags.map(&:last) + + assert_equal(%w(R-18G derp tag1 tag2 オリジナル), pixiv_tags) + assert_contains(pixiv_links, /search\.php/) end should "convert a page into a json representation" do @@ -38,5 +47,55 @@ module Sources end end end + + context "fetching source data for an old manga image" do + setup do + get_source("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=45792845", "source-pixiv-old-manga") + end + + should "get the page count" do + assert_equal(3, @site.page_count) + end + + should "get the full size image url" do + assert_equal("http://i2.pixiv.net/img18/img/ringo78/45792845_big_p0.jpg", @site.image_url) + end + end + + context "fetching source data for a new illustration" do + setup do + get_source("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46337015", "source-pixiv-new-illust") + end + + should "get the page count" do + assert_equal(1, @site.page_count) + end + + should "get the full size image url" do + assert_equal("http://i2.pixiv.net/img-original/img/2014/10/04/03/59/52/46337015_p0.png", @site.image_url) + end + end + + context "fetching source data for an old illustration" do + setup do + get_source("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=14901720", "source-pixiv-old-illust") + end + + should "get the page count" do + assert_equal(1, @site.page_count) + end + + should "get the full size image url" do + assert_equal("http://i2.pixiv.net/img18/img/evazion/14901720.png", @site.image_url) + end + + should "get the tags" do + pixiv_tags = @site.tags.map(&:first) + pixiv_links = @site.tags.map(&:last) + + assert_equal(%w(derp), pixiv_tags) + assert_contains(pixiv_links, /search\.php/) + end + end end end