From 5ba63733d3766b59ac6b47f8177e447a098cb005 Mon Sep 17 00:00:00 2001
From: evazion <noizave@gmail.com>
Date: Tue, 21 Nov 2017 13:48:07 -0600
Subject: [PATCH 1/3] download/pixiv_test.rb: fix broken pixiv download tests.

This post was deleted:

  http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46304614

Switch to this instead:

  http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46324488
---
 test/unit/downloads/pixiv_test.rb | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb
index a22ceb5fc..cd485578b 100644
--- a/test/unit/downloads/pixiv_test.rb
+++ b/test/unit/downloads/pixiv_test.rb
@@ -74,19 +74,17 @@ module Downloads
 
       context "downloading a new manga image" do
         setup do
-          @medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46304614"
-          @manga_page  = "http://www.pixiv.net/member_illust.php?mode=manga&illust_id=46304614"
-          @manga_big_p1_page = "http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=46304614&page=1"
+          @medium_page = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46324488"
+          @manga_page  = "http://www.pixiv.net/member_illust.php?mode=manga&illust_id=46324488"
+          @manga_big_p1_page = "http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=46324488&page=1"
 
-          @p0_large_thumbnail = "https://i.pximg.net/c/1200x1200/img-master/img/2014/10/02/14/21/39/46304614_p0_master1200.jpg"
-          @p1_large_thumbnail = "https://i.pximg.net/c/1200x1200/img-master/img/2014/10/02/14/21/39/46304614_p1_master1200.jpg"
-          @p0_full_size_image = "https://i.pximg.net/img-original/img/2014/10/02/14/21/39/46304614_p0.gif"
-          @p0_full_size_image_3 = "https://i.pximg.net/img-original/img/2014/10/02/14/21/39/46304614_p0.gif"
-          @p1_full_size_image = "https://i.pximg.net/img-original/img/2014/10/02/14/21/39/46304614_p1.gif"
-          @p1_full_size_image_3 = "https://i.pximg.net/img-original/img/2014/10/02/14/21/39/46304614_p1.gif"
+          @p0_large_thumbnail = "https://i.pximg.net/img-master/img/2014/10/03/18/10/20/46324488_p0_master1200.jpg"
+          @p1_large_thumbnail = "https://i.pximg.net/img-master/img/2014/10/03/18/10/20/46324488_p1_master1200.jpg"
+          @p0_full_size_image = "https://i.pximg.net/img-original/img/2014/10/03/18/10/20/46324488_p0.png"
+          @p1_full_size_image = "https://i.pximg.net/img-original/img/2014/10/03/18/10/20/46324488_p1.png"
 
-          @p0_file_size = 61_131
-          @p1_file_size = 46_818
+          @p0_file_size = 21_213
+          @p1_file_size = 24_672
         end
 
         should "download the full size image" do
@@ -98,17 +96,17 @@ module Downloads
         end
 
         should "download the full size image instead of the HTML page" do
-          assert_rewritten(@p0_full_size_image_3, @medium_page)
-          assert_rewritten(@p0_full_size_image_3, @manga_page)
-          assert_rewritten(@p1_full_size_image_3, @manga_big_p1_page)
+          assert_rewritten(@p0_full_size_image, @medium_page)
+          assert_rewritten(@p0_full_size_image, @manga_page)
+          assert_rewritten(@p1_full_size_image, @manga_big_p1_page)
           assert_downloaded(@p0_file_size, @medium_page)
           assert_downloaded(@p0_file_size, @manga_page)
           assert_downloaded(@p1_file_size, @manga_big_p1_page)
         end
 
         should "download the full size image instead of the thumbnail" do
-          assert_rewritten(@p0_full_size_image_3, @p0_large_thumbnail)
-          assert_rewritten(@p1_full_size_image_3, @p1_large_thumbnail)
+          assert_rewritten(@p0_full_size_image, @p0_large_thumbnail)
+          assert_rewritten(@p1_full_size_image, @p1_large_thumbnail)
           assert_downloaded(@p0_file_size, @p0_large_thumbnail)
           assert_downloaded(@p1_file_size, @p1_large_thumbnail)
         end

From 183a7e07e308401f74d9948b8f8c2b6dc6624e02 Mon Sep 17 00:00:00 2001
From: evazion <noizave@gmail.com>
Date: Tue, 21 Nov 2017 10:13:49 -0600
Subject: [PATCH 2/3] pixiv: fix failbooru when uploading bad pixiv id posts.

Fixes an "incompatible character encodings: UTF-8 and ASCII-8BIT"
exception when uploading a deleted Pixiv post.

Caused by the Pixiv API error message not having the encoding set to UTF-8.
---
 app/logical/pixiv_api_client.rb | 11 +++++++----
 app/views/static/error.html.erb |  4 ++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb
index b23962820..59830d78d 100644
--- a/app/logical/pixiv_api_client.rb
+++ b/app/logical/pixiv_api_client.rb
@@ -141,12 +141,13 @@ class PixivApiClient
 
     url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
     resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(query: params, headers: headers))
+    body = resp.body.force_encoding("utf-8")
 
     if resp.success?
-      json = parse_api_json(resp.body)
+      json = parse_api_json(body)
       WorksResponse.new(json["response"][0])
     else
-      raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{resp.body})")
+      raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
     end
   end
 
@@ -177,11 +178,13 @@ private
       url = "https://oauth.secure.pixiv.net/auth/token"
 
       resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers))
+      body = resp.body.force_encoding("utf-8")
+
       if resp.success?
-        json = JSON.parse(resp.body)
+        json = JSON.parse(body)
         access_token = json["response"]["access_token"]
       else
-        raise Error.new("Pixiv API access token call failed (status=#{resp.code} body=#{resp.body})")
+        raise Error.new("Pixiv API access token call failed (status=#{resp.code} body=#{body})")
       end
 
       access_token
diff --git a/app/views/static/error.html.erb b/app/views/static/error.html.erb
index e0daa2bbd..6d09563e7 100644
--- a/app/views/static/error.html.erb
+++ b/app/views/static/error.html.erb
@@ -1,7 +1,7 @@
 <% if CurrentUser.user.is_builder? && @exception.present? %>
   <h1><%= @exception.class.to_s %> exception raised</h1>
   <ul style="font-family: monospace; font-size: 1.2em; list-style-type: none;">
-    <li><%= @exception.message %></li>
+    <li><%= @exception.message.force_encoding("utf-8") %></li>
     <%- Rails.backtrace_cleaner.clean(@exception.backtrace).each do |b| -%>
       <li style="list-style-type: none;"><%= b %></li>
     <%- end -%>
@@ -9,5 +9,5 @@
 <% elsif @error_message %>
   <p><%= @error_message %></p>
 <% else %>
-  <p><%= @exception.message %></p>
+  <p><%= @exception.message.force_encoding("utf-8") %></p>
 <% end %>

From 0b5ed163feff937b978a81d4ee9b38e6e7c0b22a Mon Sep 17 00:00:00 2001
From: evazion <noizave@gmail.com>
Date: Tue, 21 Nov 2017 10:14:32 -0600
Subject: [PATCH 3/3] Fix #3395: fix pixiv API exception when uploading bad
 pixiv id images.

Fixes an exception when attempting to upload a Pixiv image from a
deleted work. The download strategy tries to fetch the source data in
the course of rewriting the URL, which fails if the work has been
deleted from Pixiv.

Raise a BadIDError and leave the URL as-is (don't rewrite it).
---
 .../downloads/rewrite_strategies/pixiv.rb     |  2 ++
 app/logical/pixiv_api_client.rb               | 18 +++++----------
 test/unit/downloads/pixiv_test.rb             | 23 +++++++++++++++++--
 test/unit/sources/pixiv_test.rb               |  8 +++++++
 4 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/app/logical/downloads/rewrite_strategies/pixiv.rb b/app/logical/downloads/rewrite_strategies/pixiv.rb
index 3d8316d1f..863258195 100644
--- a/app/logical/downloads/rewrite_strategies/pixiv.rb
+++ b/app/logical/downloads/rewrite_strategies/pixiv.rb
@@ -27,6 +27,8 @@ module Downloads
           data[:ugoira_content_type] = source.ugoira_content_type
         end
 
+        return [url, headers, data]
+      rescue PixivApiClient::BadIDError
         return [url, headers, data]
       end
 
diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb
index 59830d78d..7c9e50a73 100644
--- a/app/logical/pixiv_api_client.rb
+++ b/app/logical/pixiv_api_client.rb
@@ -19,6 +19,7 @@ class PixivApiClient
   ]
 
   class Error < Exception ; end
+  class BadIDError < Error ; end
 
   class WorksResponse
     attr_reader :json, :pages, :name, :moniker, :user_id, :page_count, :tags
@@ -142,24 +143,17 @@ class PixivApiClient
     url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
     resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(query: params, headers: headers))
     body = resp.body.force_encoding("utf-8")
+    json = JSON.parse(body)
 
     if resp.success?
-      json = parse_api_json(body)
       WorksResponse.new(json["response"][0])
+    elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
+      raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.")
     else
       raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
     end
-  end
-
-private
-  def parse_api_json(body)
-    json = JSON.parse(body)
-
-    if json["status"] != "success"
-      raise Error.new("Pixiv API call failed (status=#{json['status']} body=#{body})")
-    end
-
-    json
+  rescue JSON::ParserError
+    raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
   end
 
   def access_token
diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb
index cd485578b..1dbb5c532 100644
--- a/test/unit/downloads/pixiv_test.rb
+++ b/test/unit/downloads/pixiv_test.rb
@@ -43,8 +43,6 @@ module Downloads
         end
       end
 
-
-
       # Test a new illustration (one uploaded after 2014-09-30). New illustrations
       # must use /img-original/ for full size URLs. Old /imgXX/img/username/ style URLs
       # don't work for images uploaded after this date.
@@ -112,6 +110,27 @@ module Downloads
         end
       end
 
+      context "downloading a bad id image" do
+        setup do
+          @bad_id_full   = "https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png"
+          @bad_id_sample = "https://i.pximg.net/c/600x600/img-master/img/2017/11/22/01/06/44/65991677_p0_master1200.jpg"
+        end
+
+        should "not raise an error when rewriting the url" do
+          assert_nothing_raised { assert_not_rewritten(@bad_id_full) }
+        end
+
+        should_eventually "rewrite bad id samples to full size" do
+          assert_rewritten(@bad_id_full, @bad_id_sample)
+        end
+
+        # XXX This may fail someday. Pixiv doesn't delete bad id images right
+        # away, but they may be deleted eventually.
+        should "download the image" do
+          assert_downloaded(21440, @bad_id_full)
+        end
+      end
+
       context "downloading a ugoira" do
         setup do
           @medium_page     = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364"
diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb
index 34feca41b..58eb39aef 100644
--- a/test/unit/sources/pixiv_test.rb
+++ b/test/unit/sources/pixiv_test.rb
@@ -127,6 +127,14 @@ module Sources
         end        
       end
 
+      context "fetching source data for a deleted work" do
+        should "raise a bad id error" do
+          assert_raise(::PixivApiClient::BadIDError) do
+            get_source("https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png")
+          end
+        end
+      end
+
       context "fetching the commentary" do
         should "work when the description is blank" do
           get_source("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=65981746")