Fix #3982: Uploads: URI::InvalidURIError - URI must be ascii only.

app/logical/upload_service.rb

@@ -48,7 +48,7 @@ class UploadService
 
       @upload.update(status: "processing")
 
-      if @upload.file.nil? && Utils.is_downloadable?(source)
+      if @upload.file.nil? && @upload.source_url.present?
         @upload.file = Utils.download_for_upload(@upload)
       end
 
@@ -71,10 +71,6 @@ class UploadService
     return @post.warnings.full_messages
   end
 
-  def source
-    params[:source]
-  end
-
   def include_artist_commentary?
     params[:include_artist_commentary].to_s.truthy?
   end

app/logical/upload_service/preprocessor.rb

@@ -93,7 +93,7 @@ class UploadService
 
         if params[:file].present?
           file = params[:file]
-        elsif Utils.is_downloadable?(source)
+        elsif upload.source_url.present?
           file = Utils.download_for_upload(upload)
         end
 

app/logical/upload_service/utils.rb

@@ -210,7 +210,7 @@ class UploadService
       attempts = 0
 
       begin
-        download = Downloads::File.new(upload.source, upload.referer_url)
+        download = Downloads::File.new(upload.source_url, upload.referer_url)
         file, strategy = download.download!
 
         if !DanbooruImageResizer.validate_shell(file)

app/models/upload.rb

@@ -135,6 +135,24 @@ class Upload < ApplicationRecord
     end
   end
 
+  module SourceMethods
+    def source=(source)
+      source = source.unicode_normalize(:nfc)
+
+      # percent encode unicode characters in urls
+      if source =~ %r!\Ahttps?://!i
+        source = Addressable::URI.normalized_encode(source) rescue source
+      end
+
+      super(source)
+    end
+
+    def source_url
+      return nil unless source =~ %r!\Ahttps?://!i
+      Addressable::URI.heuristic_parse(source) rescue nil
+    end
+  end
+
   module UploaderMethods
     def uploader_name
       User.id_to_name(uploader_id)
@@ -233,6 +251,7 @@ class Upload < ApplicationRecord
   include VideoMethods
   extend SearchMethods
   include ApiMethods
+  include SourceMethods
 
   def uploader_is_not_limited
     if !uploader.can_upload?

test/models/upload_service_test.rb

@@ -46,7 +46,7 @@ class UploadServiceTest < ActiveSupport::TestCase
         setup do
           @source = "https://raikou1.donmai.us/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
           @mock_upload = mock("upload")
-          @mock_upload.stubs(:source).returns(@source)
+          @mock_upload.stubs(:source_url).returns(@source)
           @mock_upload.stubs(:referer_url).returns(nil)
           @bad_file = File.open("#{Rails.root}/test/files/test-corrupt.jpg", "rb")
           Downloads::File.any_instance.stubs(:download!).returns([@bad_file, nil])
@@ -1087,6 +1087,27 @@ class UploadServiceTest < ActiveSupport::TestCase
         assert_equal("blah", upload.post.tag_string)
       end
     end
+
+    context "with a source containing unicode characters" do
+      should "upload successfully" do
+        source1 = "https://raikou1.donmai.us/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg?one=東方&two=a%20b"
+        source2 = "https://raikou1.donmai.us/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg?one=%E6%9D%B1%E6%96%B9&two=a%20b"
+        service = subject.new(source: source1, rating: "s")
+
+        assert_nothing_raised { @upload = service.start! }
+        assert_equal(true, @upload.is_completed?)
+        assert_equal(source2, @upload.source)
+      end
+
+      should "normalize unicode characters in the source field" do
+        source1 = "poke\u0301mon" # pokémon (nfd form)
+        source2 = "pok\u00e9mon"  # pokémon (nfc form)
+        service = subject.new(source: source1, rating: "s", file: upload_file("test/files/test.jpg"))
+
+        assert_nothing_raised { @upload = service.start! }
+        assert_equal(source2, @upload.source)
+      end
+    end
   end
 
   context "#create_post_from_upload" do