diff --git a/app/logical/upload_service.rb b/app/logical/upload_service.rb index 3c02bb075..42ac90a4c 100644 --- a/app/logical/upload_service.rb +++ b/app/logical/upload_service.rb @@ -48,7 +48,7 @@ class UploadService @upload.update(status: "processing") - if @upload.file.nil? && Utils.is_downloadable?(source) + if @upload.file.nil? && @upload.source_url.present? @upload.file = Utils.download_for_upload(@upload) end @@ -71,10 +71,6 @@ class UploadService return @post.warnings.full_messages end - def source - params[:source] - end - def include_artist_commentary? params[:include_artist_commentary].to_s.truthy? end diff --git a/app/logical/upload_service/preprocessor.rb b/app/logical/upload_service/preprocessor.rb index a42e5d0cf..77fef8b96 100644 --- a/app/logical/upload_service/preprocessor.rb +++ b/app/logical/upload_service/preprocessor.rb @@ -93,7 +93,7 @@ class UploadService if params[:file].present? file = params[:file] - elsif Utils.is_downloadable?(source) + elsif upload.source_url.present? file = Utils.download_for_upload(upload) end diff --git a/app/logical/upload_service/utils.rb b/app/logical/upload_service/utils.rb index 8abe24f46..73f26cc07 100644 --- a/app/logical/upload_service/utils.rb +++ b/app/logical/upload_service/utils.rb @@ -210,7 +210,7 @@ class UploadService attempts = 0 begin - download = Downloads::File.new(upload.source, upload.referer_url) + download = Downloads::File.new(upload.source_url, upload.referer_url) file, strategy = download.download! if !DanbooruImageResizer.validate_shell(file) diff --git a/app/models/upload.rb b/app/models/upload.rb index 320e9c197..b775a2a51 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -135,6 +135,24 @@ class Upload < ApplicationRecord end end + module SourceMethods + def source=(source) + source = source.unicode_normalize(:nfc) + + # percent encode unicode characters in urls + if source =~ %r!\Ahttps?://!i + source = Addressable::URI.normalized_encode(source) rescue source + end + + super(source) + end + + def source_url + return nil unless source =~ %r!\Ahttps?://!i + Addressable::URI.heuristic_parse(source) rescue nil + end + end + module UploaderMethods def uploader_name User.id_to_name(uploader_id) @@ -233,6 +251,7 @@ class Upload < ApplicationRecord include VideoMethods extend SearchMethods include ApiMethods + include SourceMethods def uploader_is_not_limited if !uploader.can_upload? diff --git a/test/models/upload_service_test.rb b/test/models/upload_service_test.rb index e1a680ab3..eaa51bb16 100644 --- a/test/models/upload_service_test.rb +++ b/test/models/upload_service_test.rb @@ -46,7 +46,7 @@ class UploadServiceTest < ActiveSupport::TestCase setup do @source = "https://raikou1.donmai.us/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg" @mock_upload = mock("upload") - @mock_upload.stubs(:source).returns(@source) + @mock_upload.stubs(:source_url).returns(@source) @mock_upload.stubs(:referer_url).returns(nil) @bad_file = File.open("#{Rails.root}/test/files/test-corrupt.jpg", "rb") Downloads::File.any_instance.stubs(:download!).returns([@bad_file, nil]) @@ -1087,6 +1087,27 @@ class UploadServiceTest < ActiveSupport::TestCase assert_equal("blah", upload.post.tag_string) end end + + context "with a source containing unicode characters" do + should "upload successfully" do + source1 = "https://raikou1.donmai.us/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg?one=東方&two=a%20b" + source2 = "https://raikou1.donmai.us/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg?one=%E6%9D%B1%E6%96%B9&two=a%20b" + service = subject.new(source: source1, rating: "s") + + assert_nothing_raised { @upload = service.start! } + assert_equal(true, @upload.is_completed?) + assert_equal(source2, @upload.source) + end + + should "normalize unicode characters in the source field" do + source1 = "poke\u0301mon" # pokémon (nfd form) + source2 = "pok\u00e9mon" # pokémon (nfc form) + service = subject.new(source: source1, rating: "s", file: upload_file("test/files/test.jpg")) + + assert_nothing_raised { @upload = service.start! } + assert_equal(source2, @upload.source) + end + end end context "#create_post_from_upload" do