include whitecube source/rewrite strategies

app/logical/downloads/rewrite_strategies/pixiv.rb

@@ -53,10 +53,11 @@ module Downloads
       #   http://www.pixiv.net/member_illust.php?mode=big&illust_id=18557054
       #   http://www.pixiv.net/member_illust.php?mode=manga&illust_id=18557054
       #   http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=18557054&page=1
+      #   http://www.pixiv.net/whitecube/user/277898/illust/59182257
       # Plus this:
       #   i2.pixiv.net/img-inf/img/2014/09/25/00/57/24/46170939_64x64.jpg
       def rewrite_html_pages(url, headers)
-        if url =~ /illust_id=\d+/i || url =~ %r!pixiv\.net/img-inf/img/!i
+        if url =~ /illust_id=\d+/i || url =~ %r!pixiv\.net/img-inf/img/!i || url =~ /illust\/\d+/
           return [source.file_url, headers]
         else
           return [url, headers]

app/logical/pixiv_api_client.rb

@@ -6,7 +6,7 @@ class PixivApiClient
   class Error < Exception ; end
 
   class WorksResponse
-    attr_reader :json, :pages, :moniker, :page_count
+    attr_reader :json, :pages, :name, :moniker, :user_id, :page_count, :tags
     attr_reader :artist_commentary_title, :artist_commentary_desc
 
     def initialize(json)
@@ -90,10 +90,13 @@ class PixivApiClient
       # }
 
       @json = json
+      @name = json["user"]["name"]
+      @user_id = json["user"]["id"]
       @moniker = json["user"]["account"]
       @page_count = json["page_count"].to_i
       @artist_commentary_title = json["title"]
       @artist_commentary_desc = json["caption"]
+      @tags = json["tags"]
 
       if page_count > 1
         @pages = json["metadata"]["pages"].map {|x| x["image_urls"]["large"]}

app/logical/pixiv_ugoira_converter.rb

@@ -31,7 +31,7 @@ class PixivUgoiraConverter
         f.write("# timecode format v2\n")
         frame_data.each do |img|
           f.write("#{delay_sum}\n")
-          delay_sum += img["delay"]
+          delay_sum += (img["delay"] || img["delay_msec"])
         end
         f.write("#{delay_sum}\n")
         f.write("#{delay_sum}\n")

app/logical/sources/site.rb

@@ -10,7 +10,7 @@ module Sources
       :artist_commentary_desc, :to => :strategy
 
     def self.strategies
-      [Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr]
+      [Strategies::PixivWhitecube, Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr]
     end
 
     def initialize(url, options = {})

app/logical/sources/strategies/pixiv.rb

@@ -11,12 +11,13 @@ module Sources
       TIMESTAMP = '(?:[0-9]{4}/[0-9]{2}/[0-9]{2}/[0-9]{2}/[0-9]{2}/[0-9]{2})'
       EXT = "(?:jpg|jpeg|png|gif)"
 
+      WHITECUBE = /^https?:\/\/wwww\.pixiv\.net\/whitecube/i
       WEB = "^(?:https?://)?www\\.pixiv\\.net"
       I12 = "^(?:https?://)?i[0-9]+\\.pixiv\\.net"
       IMG = "^(?:https?://)?img[0-9]*\\.pixiv\\.net"
 
       def self.url_match?(url)
-        url =~ /#{WEB}|#{IMG}|#{I12}/i
+        url !~ WHITECUBE && url =~ /#{WEB}|#{IMG}|#{I12}/i
       end
 
       def referer_url

app/logical/sources/strategies/pixiv_whitecube.rb

@@ -0,0 +1,83 @@
+module Sources
+  module Strategies
+    class PixivWhitecube < Base
+      # sample: http://www.pixiv.net/whitecube/user/277898/illust/59182257
+      WHITECUBE_ILLUST = %r!^https?://www\.pixiv\.net/whitecube/user/\d+/illust/\d+!i
+
+      def self.url_match?(url)
+        url =~ WHITECUBE_ILLUST
+      end
+
+      def referer_url
+        @url
+      end
+
+      def site_name
+        "Pixiv Whitecube"
+      end
+
+      def unique_id
+        @pixiv_moniker
+      end
+
+      def fake_referer
+        "http://www.pixiv.net"
+      end
+
+      def has_artist_commentary?
+        @artist_commentary_desc.present?
+      end
+
+      def normalizable_for_artist_finder?
+        true
+      end
+
+      def normalize_for_artist_finder!
+        "http://img.pixiv.net/img/#{@moniker}/"
+      end
+
+      def get
+        @illust_id = illust_id_from_url
+        @data = query_pixiv_api(@illust_id)
+
+        @artist_name = @data.name
+        @profile_url = "https://www.pixiv.net/whitecube/user/" + @data.user_id.to_s
+        @pixiv_moniker = @data.moniker
+        @zip_url, @ugoira_frame_data = get_zip_url_from_api(@data)
+        @tags = @data.tags
+        @page_count = @data.page_count
+        @artist_commentary_title = @data.artist_commentary_title
+        @artist_commentary_desc = @data.artist_commentary_desc
+
+        is_manga = @page_count > 1
+
+        if !@zip_url
+          @image_url = @data.pages.first
+        end
+      end
+
+      def illust_id_from_url
+        # http://www.pixiv.net/whitecube/user/277898/illust/59182257
+        if url =~ %r!/whitecube/user/\d+/illust/(\d+)!
+          $1
+
+        else
+          raise Sources::Error.new("Couldn't get illust ID from URL: #{url}")
+        end
+      end
+
+      def query_pixiv_api(illust_id)
+        @data ||= PixivApiClient.new.works(illust_id)
+      end
+
+      def get_zip_url_from_api(data)
+        if data.json["metadata"] && data.json["metadata"]["zip_urls"]
+          zip_url = data.json["metadata"]["zip_urls"]["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
+          frame_data = data.json["metadata"]["frames"]
+
+          return [zip_url, frame_data]
+        end
+      end
+    end
+  end
+end

config/danbooru_default_config.rb

@@ -267,6 +267,14 @@ module Danbooru
       nil
     end
 
+    def pixiv_whitecube_login
+      nil
+    end
+
+    def pixiv_whitecube_password
+      nil
+    end
+
     def tinami_login
       nil
     end

test/fixtures/vcr_cassettes/pixiv-whitecube-ilust.yml

@@ -0,0 +1,117 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://oauth.secure.pixiv.net/auth/token
+    body:
+      encoding: US-ASCII
+      string: username=SENSITIVE&password=SENSITIVE&grant_type=password&client_id=bYGKuGVw91e0NMfPGp44euvGt59s&client_secret=HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK
+    headers:
+      Referer:
+      - http://www.pixiv.net
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx
+      Date:
+      - Tue, 27 Sep 2016 22:47:51 GMT
+      Content-Type:
+      - application/json
+      Content-Length:
+      - '352'
+      Connection:
+      - keep-alive
+      Set-Cookie:
+      - PHPSESSID=14613426_55e806b338f5a162453aee474b0d653f; expires=Tue, 27-Sep-2016
+        23:47:51 GMT; Max-Age=3600; path=/; domain=.pixiv.net; secure
+      Expires:
+      - Thu, 19 Nov 1981 08:52:00 GMT
+      Cache-Control:
+      - no-store, no-cache, must-revalidate, post-check=0, pre-check=0
+      Pragma:
+      - no-cache
+      X-Content-Type-Options:
+      - nosniff
+      Vary:
+      - Accept-Encoding
+    body:
+      encoding: UTF-8
+      string: '{"response":{"access_token":"Q6cRks2WRIlQI88HSW17xc1QdGvC-lp2rCLjtZT96VI","expires_in":3600,"token_type":"bearer","scope":"unlimited","refresh_token":"-SGkey31OzOEWAj8B2lrCPlaFkAEja95Ie5Cj6-DSVM","user":{"profile_image_urls":{"px_16x16":"http:\/\/source.pixiv.net\/common\/images\/no_profile_ss.png","px_50x50":"http:\/\/source.pixiv.net\/common\/images\/no_profile_s.png","px_170x170":"http:\/\/source.pixiv.net\/common\/images\/no_profile.png"},"id":"14613426","name":"SENSITIVE","account":"SENSITIVE","is_premium":false,"x_restrict":2,"is_mail_authorized":true}}}'
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 22:47:51 GMT
+- request:
+    method: get
+    uri: https://public-api.secure.pixiv.net/v1/works/59182257.json?image_sizes=large&include_stats=true
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Referer:
+      - http://www.pixiv.net
+      User-Agent:
+      - dior/2.105.0
+      Content-Type:
+      - application/x-www-form-urlencoded
+      Authorization:
+      - Bearer Q6cRks2WRIlQI88HSW17xc1QdGvC-lp2rCLjtZT96VI
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx
+      Date:
+      - Tue, 27 Sep 2016 22:47:52 GMT
+      Content-Type:
+      - application/json
+      Content-Length:
+      - '622'
+      Connection:
+      - keep-alive
+      X-Content-Type-Options:
+      - nosniff
+      Vary:
+      - Accept-Encoding
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        eyJzdGF0dXMiOiJzdWNjZXNzIiwicmVzcG9uc2UiOlt7ImlkIjo1OTE4MjI1
+        NywidGl0bGUiOiLvvLPvvKTntKsiLCJjYXB0aW9uIjoi57WQ55WM44OB44O8
+        44OgIiwidGFncyI6WyLmnbHmlrkiLCLlhavpm7LntKsiXSwidG9vbHMiOlsi
+        Q0xJUCBTVFVESU8gUEFJTlQiLCJBelBhaW50ZXIyIiwiQXpEcmF3aW5nMiJd
+        LCJpbWFnZV91cmxzIjp7ImxhcmdlIjoiaHR0cDovL2kyLnBpeGl2Lm5ldC9p
+        bWctb3JpZ2luYWwvaW1nLzIwMTYvMDkvMjYvMjEvMzAvNDEvNTkxODIyNTdf
+        cDAuanBnIn0sIndpZHRoIjo4MDAsImhlaWdodCI6ODAwLCJzdGF0cyI6eyJz
+        Y29yZWRfY291bnQiOjQwLCJzY29yZSI6Mzk5LCJ2aWV3c19jb3VudCI6Mzk2
+        LCJmYXZvcml0ZWRfY291bnQiOnsicHVibGljIjoxOSwicHJpdmF0ZSI6Mn0s
+        ImNvbW1lbnRlZF9jb3VudCI6NX0sInB1YmxpY2l0eSI6MCwiYWdlX2xpbWl0
+        IjoiYWxsLWFnZSIsImNyZWF0ZWRfdGltZSI6IjIwMTYtMDktMjYgMjE6MzA6
+        NDEiLCJyZXVwbG9hZGVkX3RpbWUiOiIyMDE2LTA5LTI2IDIxOjMwOjQxIiwi
+        dXNlciI6eyJpZCI6Mjc3ODk4LCJhY2NvdW50IjoibW9ub2NhdGllbnVzIiwi
+        bmFtZSI6ImNhdG8iLCJpc19mb2xsb3dpbmciOmZhbHNlLCJpc19mb2xsb3dl
+        ciI6ZmFsc2UsImlzX2ZyaWVuZCI6ZmFsc2UsImlzX3ByZW1pdW0iOm51bGws
+        InByb2ZpbGVfaW1hZ2VfdXJscyI6eyJweF81MHg1MCI6Imh0dHA6Ly9pMi5w
+        aXhpdi5uZXQvdXNlci1wcm9maWxlL2ltZy8yMDE2LzA5LzA3LzIyLzQ2LzIx
+        LzExNDY1NzAxXzY4YmRkZTIyMDI2M2U5NzgwZGE4MjQ1YzkzMzQ4Yzc4XzUw
+        LmpwZyJ9LCJzdGF0cyI6bnVsbCwicHJvZmlsZSI6bnVsbH0sImlzX21hbmdh
+        IjpmYWxzZSwiaXNfbGlrZWQiOmZhbHNlLCJmYXZvcml0ZV9pZCI6MCwicGFn
+        ZV9jb3VudCI6MSwiYm9va19zdHlsZSI6Im5vbmUiLCJ0eXBlIjoiaWxsdXN0
+        cmF0aW9uIiwibWV0YWRhdGEiOm51bGwsImNvbnRlbnRfdHlwZSI6bnVsbH1d
+        LCJjb3VudCI6MX0=
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 22:47:52 GMT
+recorded_with: VCR 2.9.3

test/unit/sources/pixiv_test.rb

@@ -18,9 +18,23 @@ module Sources
     end
 
     context "in all cases" do
+      context "A whitecube page" do
+        setup do
+          VCR.use_cassette("pixiv-whitecube-ilust", :record => :none) do
+            @site = Sources::Site.new("https://www.pixiv.net/whitecube/user/277898/illust/59182257")
+            @site.get
+            @image_urls = @site.image_urls
+          end
+        end
+
+        should "1234 get all the image urls" do
+          assert_equal(["http://i2.pixiv.net/img-original/img/2016/09/26/21/30/41/59182257_p0.jpg"], @image_urls)
+        end
+      end
+
       context "A gallery page" do
         setup do
-          VCR.use_cassette("pixiv-gallery", :record => :once) do
+          VCR.use_cassette("pixiv-gallery", :record => :none) do
             @site = Sources::Site.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=49270482")
             @site.get
             @image_urls = @site.image_urls
@@ -34,7 +48,7 @@ module Sources
 
       context "An ugoira source site for pixiv" do
         setup do
-          VCR.use_cassette("ugoira-converter", :record => :once) do
+          VCR.use_cassette("ugoira-converter", :record => :none) do
             @site = Sources::Site.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654")
             @site.get
           end