diff --git a/app/logical/sources/strategies/pixiv.rb b/app/logical/sources/strategies/pixiv.rb index 4edfbef3e..e13ee21fd 100644 --- a/app/logical/sources/strategies/pixiv.rb +++ b/app/logical/sources/strategies/pixiv.rb @@ -1,4 +1,47 @@ -require 'csv' +# Pixiv +# +# * https://i.pximg.net/img-original/img/2014/10/03/18/10/20/46324488_p0.png +# +# * https://i.pximg.net/c/250x250_80_a2/img-master/img/2014/10/29/09/27/19/46785915_p0_square1200.jpg +# * https://i.pximg.net/img-master/img/2014/10/03/18/10/20/46324488_p0_master1200.jpg +# +# * https://www.pixiv.net/member_illust.php?mode=medium&illust_id=46324488 +# * https://www.pixiv.net/member_illust.php?mode=manga&illust_id=46324488 +# * https://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=46324488&page=0 +# +# * https://www.pixiv.net/member.php?id=339253 +# * https://www.pixiv.net/member_illust.php?id=339253&type=illust +# * https://www.pixiv.net/u/9202877 +# * https://www.pixiv.net/stacc/noizave +# * http://www.pixiv.me/noizave +# +# Fanbox +# +# * https://fanbox.pixiv.net/images/post/39714/JvjJal8v1yLgc5DPyEI05YpT.png +# * https://pixiv.pximg.net/fanbox/public/images/creator/1566167/profile/Ix6bnJmTaOAFZhXHLbWyIY1e.jpeg +# +# * https://pixiv.pximg.net/c/400x400_90_a2_g5/fanbox/public/images/creator/1566167/profile/Ix6bnJmTaOAFZhXHLbWyIY1e.jpeg +# * https://pixiv.pximg.net/c/1200x630_90_a2_g5/fanbox/public/images/post/186919/cover/VCI1Mcs2rbmWPg0mmiTisovn.jpeg +# +# * https://www.pixiv.net/fanbox/creator/1566167/post/39714 +# * https://www.pixiv.net/fanbox/creator/1566167 +# +# Novels +# +# * https://i.pximg.net/novel-cover-original/img/2019/01/14/01/15/05/10617324_d84daae89092d96bbe66efafec136e42.jpg +# * https://i.pximg.net/c/600x600/novel-cover-master/img/2019/01/14/01/15/05/10617324_d84daae89092d96bbe66efafec136e42_master1200.jpg +# * https://img-novel.pximg.net/img-novel/work_main/XtFbt7gsymsvyaG45lZ8/1554.jpg?20190107110435 +# +# * https://www.pixiv.net/novel/show.php?id=10617324 +# * https://novel.pixiv.net/works/1554 +# +# Sketch +# +# * https://img-sketch.pixiv.net/uploads/medium/file/4463372/8906921629213362989.jpg +# * https://img-sketch.pximg.net/c!/w=540,f=webp:jpeg/uploads/medium/file/4463372/8906921629213362989.jpg +# * https://sketch.pixiv.net/items/1588346448904706151 +# * https://sketch.pixiv.net/@0125840 +# module Sources module Strategies @@ -218,37 +261,45 @@ module Sources # even though it makes sense to reference page_url here, it will only look # at (url, referer_url). def illust_id - # http://img18.pixiv.net/img/evazion/14901720.png - # - # http://i2.pixiv.net/img18/img/evazion/14901720.png - # http://i2.pixiv.net/img18/img/evazion/14901720_m.png - # http://i2.pixiv.net/img18/img/evazion/14901720_s.png - # http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png - # http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png - # - # http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg - # http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_s.png - # - # http://i1.pixiv.net/c/600x600/img-master/img/2014/10/02/13/51/23/46304396_p0_master1200.jpg - # http://i1.pixiv.net/img-original/img/2014/10/02/13/51/23/46304396_p0.png - # - # http://i1.pixiv.net/img-zip-ugoira/img/2014/10/03/17/29/16/46323924_ugoira1920x1080.zip - if url =~ %r!/(\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i - return $1 - end + return nil if novel_id.present? || fanbox_id.present? - [url, referer_url].each do |x| + parsed_urls.each do |url| # http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054 # http://www.pixiv.net/member_illust.php?mode=big&illust_id=18557054 # http://www.pixiv.net/member_illust.php?mode=manga&illust_id=18557054 # http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=18557054&page=1 - if x =~ /illust_id=(\d+)/i - return $1 - end + if url.host == "www.pixiv.net" && url.path == "/member_illust.php" && url.query_values["illust_id"].present? + return url.query_values["illust_id"].to_i # http://www.pixiv.net/i/18557054 - if x =~ %r!pixiv\.net/i/(\d+)!i - return $1 + elsif url.host == "www.pixiv.net" && url.path =~ %r!\A/i/(?\d+)\z!i + return $~[:illust_id].to_i + + # http://img18.pixiv.net/img/evazion/14901720.png + # http://i2.pixiv.net/img18/img/evazion/14901720.png + # http://i2.pixiv.net/img18/img/evazion/14901720_m.png + # http://i2.pixiv.net/img18/img/evazion/14901720_s.png + # http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png + # http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png + elsif url.host =~ %r!\A(?:i\d+|img\d+)\.pixiv\.net\z!i && + url.path =~ %r!\A(?:/img\d+)?/img/\w+/(?\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i + return $~[:illust_id].to_i + + # http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg + # http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_s.png + # http://i1.pixiv.net/c/600x600/img-master/img/2014/10/02/13/51/23/46304396_p0_master1200.jpg + # http://i1.pixiv.net/img-original/img/2014/10/02/13/51/23/46304396_p0.png + # http://i1.pixiv.net/img-zip-ugoira/img/2014/10/03/17/29/16/46323924_ugoira1920x1080.zip + # https://i.pximg.net/img-original/img/2014/10/03/18/10/20/46324488_p0.png + # https://i.pximg.net/img-master/img/2014/10/03/18/10/20/46324488_p0_master1200.jpg + # + # but not: + # + # https://i.pximg.net/novel-cover-original/img/2019/01/14/01/15/05/10617324_d84daae89092d96bbe66efafec136e42.jpg + # https://img-sketch.pixiv.net/uploads/medium/file/4463372/8906921629213362989.jpg + elsif url.host =~ %r!\A(?:i\.pximg\.net|i\d+\.pixiv\.net)\z!i && + url.path =~ %r!\A(/c/\w+)?/img-[a-z-]+/img/\d{4}(?:/\d{2}){5}/(?\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i + return $~[:illust_id].to_i end end diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index 36c161e4b..d2c917061 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -1438,79 +1438,9 @@ class PostTest < ActiveSupport::TestCase context "that is from pixiv" do should "save the pixiv id" do - @post.update(source: "https://img18.pixiv.net/img/evazion/14901720.png") - assert_equal(14901720, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://img18.pixiv.net/img/evazion/14901720.png") - assert_equal(14901720, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://i2.pixiv.net/img18/img/evazion/14901720.png") - assert_equal(14901720, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://i2.pixiv.net/img18/img/evazion/14901720_m.png") - assert_equal(14901720, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://i2.pixiv.net/img18/img/evazion/14901720_s.png") - assert_equal(14901720, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_s.png") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://i1.pixiv.net/c/600x600/img-master/img/2014/10/02/13/51/23/46304396_p0_master1200.jpg") - assert_equal(46304396, @post.pixiv_id) - @post.pixiv_id = nil - @post.update(source: "http://i1.pixiv.net/img-original/img/2014/10/02/13/51/23/46304396_p0.png") assert_equal(46304396, @post.pixiv_id) @post.pixiv_id = nil - - @post.update(source: "http://i1.pixiv.net/img-zip-ugoira/img/2014/10/03/17/29/16/46323924_ugoira1920x1080.zip") - assert_equal(46323924, @post.pixiv_id) - @post.pixiv_id = nil - - - - @post.update(source: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://www.pixiv.net/member_illust.php?mode=big&illust_id=18557054") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://www.pixiv.net/member_illust.php?mode=manga&illust_id=18557054") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=18557054") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil - - @post.update(source: "http://www.pixiv.net/i/18557054") - assert_equal(18557054, @post.pixiv_id) - @post.pixiv_id = nil end end diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index 259018806..ecdda1e13 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -2,9 +2,20 @@ require 'test_helper' module Sources class PixivTest < ActiveSupport::TestCase + def assert_illust_id(illust_id, url) + site = Sources::Strategies.find(url) + assert_equal(illust_id, site.illust_id) + assert_nothing_raised { site.to_h } + end + + def assert_nil_illust_id(url) + site = Sources::Strategies.find(url) + assert_nil(site.illust_id) + end + def get_source(source) @site = Sources::Strategies.find(source) - + @site rescue Net::OpenTimeout skip "Remote connection to #{source} failed" @@ -19,23 +30,8 @@ module Sources save_pixiv_tokens! super end - + context "in all cases" do - context "A touch page" do - setup do - @site = Sources::Strategies.find("http://touch.pixiv.net/member_illust.php?mode=medium&illust_id=59687915") - @image_urls = @site.image_urls - end - - should "get all the image urls" do - expected_urls = [ - "https://i.pximg.net/img-original/img/2016/10/29/17/13/23/59687915_p0.png", - "https://i.pximg.net/img-original/img/2016/10/29/17/13/23/59687915_p1.png" - ].sort - assert_equal(expected_urls, @image_urls.sort) - end - end - context "A gallery page" do setup do @site = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=49270482") @@ -292,6 +288,51 @@ module Sources assert_includes(source.profile_urls, "https://www.pixiv.net/stacc/uroobnad") end end + + context "parsing illust ids" do + should "parse ids from illust urls" do + assert_illust_id(46324488, "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=46324488") + assert_illust_id(46324488, "https://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=46324488&page=0") + assert_illust_id(46324488, "https://i.pximg.net/img-original/img/2014/10/03/18/10/20/46324488_p0.png") + assert_illust_id(46324488, "https://i.pximg.net/img-master/img/2014/10/03/18/10/20/46324488_p0_master1200.jpg") + + assert_illust_id(46785915, "https://i.pximg.net/c/250x250_80_a2/img-master/img/2014/10/29/09/27/19/46785915_p0_square1200.jpg") + + assert_illust_id(46323924, "http://i1.pixiv.net/img-zip-ugoira/img/2014/10/03/17/29/16/46323924_ugoira1920x1080.zip") + assert_illust_id(46304396, "http://i1.pixiv.net/img-original/img/2014/10/02/13/51/23/46304396_p0.png") + assert_illust_id(46304396, "http://i1.pixiv.net/c/600x600/img-master/img/2014/10/02/13/51/23/46304396_p0_master1200.jpg") + + assert_illust_id(14901720, "http://img18.pixiv.net/img/evazion/14901720.png") + assert_illust_id(14901720, "http://i2.pixiv.net/img18/img/evazion/14901720.png") + assert_illust_id(14901720, "http://i2.pixiv.net/img18/img/evazion/14901720_m.png") + assert_illust_id(14901720, "http://i2.pixiv.net/img18/img/evazion/14901720_s.png") + + assert_illust_id(18557054, "http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png") + assert_illust_id(18557054, "http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png") + assert_illust_id(18557054, "http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg") + assert_illust_id(18557054, "http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_s.png") + assert_illust_id(18557054, "http://www.pixiv.net/i/18557054") + end + + should "not misparse ids from fanbox urls" do + assert_nil_illust_id("https://fanbox.pixiv.net/images/post/39714/JvjJal8v1yLgc5DPyEI05YpT.png") + assert_nil_illust_id("https://pixiv.pximg.net/fanbox/public/images/creator/1566167/profile/Ix6bnJmTaOAFZhXHLbWyIY1e.jpeg") + assert_nil_illust_id("https://pixiv.pximg.net/c/400x400_90_a2_g5/fanbox/public/images/creator/1566167/profile/Ix6bnJmTaOAFZhXHLbWyIY1e.jpeg") + assert_nil_illust_id("https://pixiv.pximg.net/c/1200x630_90_a2_g5/fanbox/public/images/post/186919/cover/VCI1Mcs2rbmWPg0mmiTisovn.jpeg") + end + + should "not misparse ids from sketch urls" do + assert_nil_illust_id("https://img-sketch.pixiv.net/uploads/medium/file/4463372/8906921629213362989.jpg") + assert_nil_illust_id("https://img-sketch.pximg.net/c!/w=540,f=webp:jpeg/uploads/medium/file/4463372/8906921629213362989.jpg") + assert_nil_illust_id("https://sketch.pixiv.net/items/1588346448904706151") + end + + should "not misparse ids from novel urls" do + assert_nil_illust_id("https://i.pximg.net/novel-cover-original/img/2019/01/14/01/15/05/10617324_d84daae89092d96bbe66efafec136e42.jpg") + assert_nil_illust_id("https://i.pximg.net/c/600x600/novel-cover-master/img/2019/01/14/01/15/05/10617324_d84daae89092d96bbe66efafec136e42_master1200.jpg") + assert_nil_illust_id("https://www.pixiv.net/novel/show.php?id=10617324") + end + end end end end