diff --git a/app/logical/downloads/rewrite_strategies/pixiv.rb b/app/logical/downloads/rewrite_strategies/pixiv.rb index 6eae53558..339a71bf3 100644 --- a/app/logical/downloads/rewrite_strategies/pixiv.rb +++ b/app/logical/downloads/rewrite_strategies/pixiv.rb @@ -53,10 +53,11 @@ module Downloads # http://www.pixiv.net/member_illust.php?mode=big&illust_id=18557054 # http://www.pixiv.net/member_illust.php?mode=manga&illust_id=18557054 # http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=18557054&page=1 + # http://www.pixiv.net/whitecube/user/277898/illust/59182257 # Plus this: # i2.pixiv.net/img-inf/img/2014/09/25/00/57/24/46170939_64x64.jpg def rewrite_html_pages(url, headers) - if url =~ /illust_id=\d+/i || url =~ %r!pixiv\.net/img-inf/img/!i + if url =~ /illust_id=\d+/i || url =~ %r!pixiv\.net/img-inf/img/!i || url =~ /illust\/\d+/ return [source.file_url, headers] else return [url, headers] diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb index 869e94eae..25b080317 100644 --- a/app/logical/pixiv_api_client.rb +++ b/app/logical/pixiv_api_client.rb @@ -6,7 +6,7 @@ class PixivApiClient class Error < Exception ; end class WorksResponse - attr_reader :json, :pages, :moniker, :page_count + attr_reader :json, :pages, :name, :moniker, :user_id, :page_count, :tags attr_reader :artist_commentary_title, :artist_commentary_desc def initialize(json) @@ -90,10 +90,13 @@ class PixivApiClient # } @json = json + @name = json["user"]["name"] + @user_id = json["user"]["id"] @moniker = json["user"]["account"] @page_count = json["page_count"].to_i @artist_commentary_title = json["title"] @artist_commentary_desc = json["caption"] + @tags = json["tags"] if page_count > 1 @pages = json["metadata"]["pages"].map {|x| x["image_urls"]["large"]} diff --git a/app/logical/pixiv_ugoira_converter.rb b/app/logical/pixiv_ugoira_converter.rb index 21d75ea10..b7d1c899c 100644 --- a/app/logical/pixiv_ugoira_converter.rb +++ b/app/logical/pixiv_ugoira_converter.rb @@ -31,7 +31,7 @@ class PixivUgoiraConverter f.write("# timecode format v2\n") frame_data.each do |img| f.write("#{delay_sum}\n") - delay_sum += img["delay"] + delay_sum += (img["delay"] || img["delay_msec"]) end f.write("#{delay_sum}\n") f.write("#{delay_sum}\n") diff --git a/app/logical/sources/site.rb b/app/logical/sources/site.rb index aa4e39b4c..d30eb3e3d 100644 --- a/app/logical/sources/site.rb +++ b/app/logical/sources/site.rb @@ -10,7 +10,7 @@ module Sources :artist_commentary_desc, :to => :strategy def self.strategies - [Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr] + [Strategies::PixivWhitecube, Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr] end def initialize(url, options = {}) diff --git a/app/logical/sources/strategies/pixiv.rb b/app/logical/sources/strategies/pixiv.rb index ec022b6ae..310d50e8d 100644 --- a/app/logical/sources/strategies/pixiv.rb +++ b/app/logical/sources/strategies/pixiv.rb @@ -11,12 +11,13 @@ module Sources TIMESTAMP = '(?:[0-9]{4}/[0-9]{2}/[0-9]{2}/[0-9]{2}/[0-9]{2}/[0-9]{2})' EXT = "(?:jpg|jpeg|png|gif)" + WHITECUBE = /^https?:\/\/wwww\.pixiv\.net\/whitecube/i WEB = "^(?:https?://)?www\\.pixiv\\.net" I12 = "^(?:https?://)?i[0-9]+\\.pixiv\\.net" IMG = "^(?:https?://)?img[0-9]*\\.pixiv\\.net" def self.url_match?(url) - url =~ /#{WEB}|#{IMG}|#{I12}/i + url !~ WHITECUBE && url =~ /#{WEB}|#{IMG}|#{I12}/i end def referer_url diff --git a/app/logical/sources/strategies/pixiv_whitecube.rb b/app/logical/sources/strategies/pixiv_whitecube.rb new file mode 100644 index 000000000..683892b77 --- /dev/null +++ b/app/logical/sources/strategies/pixiv_whitecube.rb @@ -0,0 +1,83 @@ +module Sources + module Strategies + class PixivWhitecube < Base + # sample: http://www.pixiv.net/whitecube/user/277898/illust/59182257 + WHITECUBE_ILLUST = %r!^https?://www\.pixiv\.net/whitecube/user/\d+/illust/\d+!i + + def self.url_match?(url) + url =~ WHITECUBE_ILLUST + end + + def referer_url + @url + end + + def site_name + "Pixiv Whitecube" + end + + def unique_id + @pixiv_moniker + end + + def fake_referer + "http://www.pixiv.net" + end + + def has_artist_commentary? + @artist_commentary_desc.present? + end + + def normalizable_for_artist_finder? + true + end + + def normalize_for_artist_finder! + "http://img.pixiv.net/img/#{@moniker}/" + end + + def get + @illust_id = illust_id_from_url + @data = query_pixiv_api(@illust_id) + + @artist_name = @data.name + @profile_url = "https://www.pixiv.net/whitecube/user/" + @data.user_id.to_s + @pixiv_moniker = @data.moniker + @zip_url, @ugoira_frame_data = get_zip_url_from_api(@data) + @tags = @data.tags + @page_count = @data.page_count + @artist_commentary_title = @data.artist_commentary_title + @artist_commentary_desc = @data.artist_commentary_desc + + is_manga = @page_count > 1 + + if !@zip_url + @image_url = @data.pages.first + end + end + + def illust_id_from_url + # http://www.pixiv.net/whitecube/user/277898/illust/59182257 + if url =~ %r!/whitecube/user/\d+/illust/(\d+)! + $1 + + else + raise Sources::Error.new("Couldn't get illust ID from URL: #{url}") + end + end + + def query_pixiv_api(illust_id) + @data ||= PixivApiClient.new.works(illust_id) + end + + def get_zip_url_from_api(data) + if data.json["metadata"] && data.json["metadata"]["zip_urls"] + zip_url = data.json["metadata"]["zip_urls"]["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip") + frame_data = data.json["metadata"]["frames"] + + return [zip_url, frame_data] + end + end + end + end +end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 3a2f9f498..d6d4d9e8d 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -267,6 +267,14 @@ module Danbooru nil end + def pixiv_whitecube_login + nil + end + + def pixiv_whitecube_password + nil + end + def tinami_login nil end diff --git a/test/fixtures/vcr_cassettes/pixiv-whitecube-ilust.yml b/test/fixtures/vcr_cassettes/pixiv-whitecube-ilust.yml new file mode 100644 index 000000000..d76db1040 --- /dev/null +++ b/test/fixtures/vcr_cassettes/pixiv-whitecube-ilust.yml @@ -0,0 +1,117 @@ +--- +http_interactions: +- request: + method: post + uri: https://oauth.secure.pixiv.net/auth/token + body: + encoding: US-ASCII + string: username=SENSITIVE&password=SENSITIVE&grant_type=password&client_id=bYGKuGVw91e0NMfPGp44euvGt59s&client_secret=HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK + headers: + Referer: + - http://www.pixiv.net + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Server: + - nginx + Date: + - Tue, 27 Sep 2016 22:47:51 GMT + Content-Type: + - application/json + Content-Length: + - '352' + Connection: + - keep-alive + Set-Cookie: + - PHPSESSID=14613426_55e806b338f5a162453aee474b0d653f; expires=Tue, 27-Sep-2016 + 23:47:51 GMT; Max-Age=3600; path=/; domain=.pixiv.net; secure + Expires: + - Thu, 19 Nov 1981 08:52:00 GMT + Cache-Control: + - no-store, no-cache, must-revalidate, post-check=0, pre-check=0 + Pragma: + - no-cache + X-Content-Type-Options: + - nosniff + Vary: + - Accept-Encoding + body: + encoding: UTF-8 + string: '{"response":{"access_token":"Q6cRks2WRIlQI88HSW17xc1QdGvC-lp2rCLjtZT96VI","expires_in":3600,"token_type":"bearer","scope":"unlimited","refresh_token":"-SGkey31OzOEWAj8B2lrCPlaFkAEja95Ie5Cj6-DSVM","user":{"profile_image_urls":{"px_16x16":"http:\/\/source.pixiv.net\/common\/images\/no_profile_ss.png","px_50x50":"http:\/\/source.pixiv.net\/common\/images\/no_profile_s.png","px_170x170":"http:\/\/source.pixiv.net\/common\/images\/no_profile.png"},"id":"14613426","name":"SENSITIVE","account":"SENSITIVE","is_premium":false,"x_restrict":2,"is_mail_authorized":true}}}' + http_version: + recorded_at: Tue, 27 Sep 2016 22:47:51 GMT +- request: + method: get + uri: https://public-api.secure.pixiv.net/v1/works/59182257.json?image_sizes=large&include_stats=true + body: + encoding: US-ASCII + string: '' + headers: + Referer: + - http://www.pixiv.net + User-Agent: + - dior/2.105.0 + Content-Type: + - application/x-www-form-urlencoded + Authorization: + - Bearer Q6cRks2WRIlQI88HSW17xc1QdGvC-lp2rCLjtZT96VI + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Server: + - nginx + Date: + - Tue, 27 Sep 2016 22:47:52 GMT + Content-Type: + - application/json + Content-Length: + - '622' + Connection: + - keep-alive + X-Content-Type-Options: + - nosniff + Vary: + - Accept-Encoding + body: + encoding: ASCII-8BIT + string: !binary |- + eyJzdGF0dXMiOiJzdWNjZXNzIiwicmVzcG9uc2UiOlt7ImlkIjo1OTE4MjI1 + NywidGl0bGUiOiLvvLPvvKTntKsiLCJjYXB0aW9uIjoi57WQ55WM44OB44O8 + 44OgIiwidGFncyI6WyLmnbHmlrkiLCLlhavpm7LntKsiXSwidG9vbHMiOlsi + Q0xJUCBTVFVESU8gUEFJTlQiLCJBelBhaW50ZXIyIiwiQXpEcmF3aW5nMiJd + LCJpbWFnZV91cmxzIjp7ImxhcmdlIjoiaHR0cDovL2kyLnBpeGl2Lm5ldC9p + bWctb3JpZ2luYWwvaW1nLzIwMTYvMDkvMjYvMjEvMzAvNDEvNTkxODIyNTdf + cDAuanBnIn0sIndpZHRoIjo4MDAsImhlaWdodCI6ODAwLCJzdGF0cyI6eyJz + Y29yZWRfY291bnQiOjQwLCJzY29yZSI6Mzk5LCJ2aWV3c19jb3VudCI6Mzk2 + LCJmYXZvcml0ZWRfY291bnQiOnsicHVibGljIjoxOSwicHJpdmF0ZSI6Mn0s + ImNvbW1lbnRlZF9jb3VudCI6NX0sInB1YmxpY2l0eSI6MCwiYWdlX2xpbWl0 + IjoiYWxsLWFnZSIsImNyZWF0ZWRfdGltZSI6IjIwMTYtMDktMjYgMjE6MzA6 + NDEiLCJyZXVwbG9hZGVkX3RpbWUiOiIyMDE2LTA5LTI2IDIxOjMwOjQxIiwi + dXNlciI6eyJpZCI6Mjc3ODk4LCJhY2NvdW50IjoibW9ub2NhdGllbnVzIiwi + bmFtZSI6ImNhdG8iLCJpc19mb2xsb3dpbmciOmZhbHNlLCJpc19mb2xsb3dl + ciI6ZmFsc2UsImlzX2ZyaWVuZCI6ZmFsc2UsImlzX3ByZW1pdW0iOm51bGws + InByb2ZpbGVfaW1hZ2VfdXJscyI6eyJweF81MHg1MCI6Imh0dHA6Ly9pMi5w + aXhpdi5uZXQvdXNlci1wcm9maWxlL2ltZy8yMDE2LzA5LzA3LzIyLzQ2LzIx + LzExNDY1NzAxXzY4YmRkZTIyMDI2M2U5NzgwZGE4MjQ1YzkzMzQ4Yzc4XzUw + LmpwZyJ9LCJzdGF0cyI6bnVsbCwicHJvZmlsZSI6bnVsbH0sImlzX21hbmdh + IjpmYWxzZSwiaXNfbGlrZWQiOmZhbHNlLCJmYXZvcml0ZV9pZCI6MCwicGFn + ZV9jb3VudCI6MSwiYm9va19zdHlsZSI6Im5vbmUiLCJ0eXBlIjoiaWxsdXN0 + cmF0aW9uIiwibWV0YWRhdGEiOm51bGwsImNvbnRlbnRfdHlwZSI6bnVsbH1d + LCJjb3VudCI6MX0= + http_version: + recorded_at: Tue, 27 Sep 2016 22:47:52 GMT +recorded_with: VCR 2.9.3 diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index c065599d4..d430a48df 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -18,9 +18,23 @@ module Sources end context "in all cases" do + context "A whitecube page" do + setup do + VCR.use_cassette("pixiv-whitecube-ilust", :record => :none) do + @site = Sources::Site.new("https://www.pixiv.net/whitecube/user/277898/illust/59182257") + @site.get + @image_urls = @site.image_urls + end + end + + should "1234 get all the image urls" do + assert_equal(["http://i2.pixiv.net/img-original/img/2016/09/26/21/30/41/59182257_p0.jpg"], @image_urls) + end + end + context "A gallery page" do setup do - VCR.use_cassette("pixiv-gallery", :record => :once) do + VCR.use_cassette("pixiv-gallery", :record => :none) do @site = Sources::Site.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=49270482") @site.get @image_urls = @site.image_urls @@ -34,7 +48,7 @@ module Sources context "An ugoira source site for pixiv" do setup do - VCR.use_cassette("ugoira-converter", :record => :once) do + VCR.use_cassette("ugoira-converter", :record => :none) do @site = Sources::Site.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654") @site.get end