jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

uploads: fix broken tests.

Browse Source 
* Fix broken upload tests.
* Fix uploads to return an error if both a file and a source are given
  at the same time, or if neither are given. Also fix the error message
  in this case so that it doesn't include "base" at the start of the string.
* Fix uploads to percent-encode any Unicode characters in the source URL.
* Add a max filesize validation to media assets.
This commit is contained in:
evazion
2022-01-29 04:38:47 -06:00
parent 5d0c14d2bd
commit 11b7bcac91
9 changed files with 248 additions and 611 deletions
Download Patch File Download Diff File Expand all files Collapse all files

362
test/unit/upload_service_test.rb
View File

@@ -1,213 +1,6 @@
require 'test_helper'
class UploadServiceTest < ActiveSupport::TestCase
UGOIRA_CONTEXT = {
"ugoira" => {
"frame_data" => [
{"file" => "000000.jpg", "delay" => 200},
{"file" => "000001.jpg", "delay" => 200},
{"file" => "000002.jpg", "delay" => 200},
{"file" => "000003.jpg", "delay" => 200},
{"file" => "000004.jpg", "delay" => 250}
],
"content_type" => "image/jpeg"
}
}
def assert_file_exists(upload, variant)
assert_nothing_raised { upload.media_asset.variant(variant).open_file }
end
def assert_file_does_not_exist(upload, variant)
assert_raise { upload.media_asset.variant(variant).open_file }
end
context "::Utils" do
context ".process_file" do
setup do
@upload = FactoryBot.build(:jpg_upload)
end
should "run" do
UploadService::Utils.process_file(@upload, @upload.file.tempfile)
assert_equal("jpg", @upload.file_ext)
assert_equal(28086, @upload.file_size)
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
assert_equal(335, @upload.image_height)
assert_equal(500, @upload.image_width)
end
should "create a media asset" do
UploadService::Utils.process_file(@upload, @upload.file.tempfile)
@media_asset = @upload.media_asset
assert_not_nil(@media_asset)
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @media_asset.md5)
assert_equal("jpg", @media_asset.file_ext)
assert_equal(28086, @media_asset.file_size)
assert_equal(500, @media_asset.image_width)
assert_equal(335, @media_asset.image_height)
metadata = @media_asset.media_metadata.metadata
assert_equal(91, metadata.count)
end
end
end
context "::Preprocessor" do
subject { UploadService::Preprocessor }
context "#start!" do
setup do
CurrentUser.user = travel_to(1.month.ago) do
FactoryBot.create(:user)
end
CurrentUser.ip_addr = "127.0.0.1"
end
teardown do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
context "for twitter" do
setup do
skip "Twitter credentials not configured" unless Sources::Strategies::Twitter.enabled?
@source = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:large"
@ref = "https://twitter.com/nounproject/status/540944400767922176"
end
should "download the file" do
@service = UploadService::Preprocessor.new(source: @source, referer_url: @ref)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(9800, @upload.file_size)
assert_equal("png", @upload.file_ext)
assert_equal("f5fe24f3a3a13885285f6627e04feec9", @upload.md5)
assert_file_exists(@upload, :preview)
assert_file_exists(@upload, :original)
end
end
context "for pixiv" do
setup do
skip "Pixiv credentials not configured" unless Sources::Strategies::Pixiv.enabled?
@source = "https://i.pximg.net/img-original/img/2014/10/29/09/27/19/46785915_p0.jpg"
@ref = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=46785915"
end
should "download the file" do
@service = UploadService::Preprocessor.new(source: @source, referer_url: @ref)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(317733, @upload.file_size)
assert_equal("jpg", @upload.file_ext)
assert_equal("4c71da5638b897aa6da1150e742e2982", @upload.md5)
assert_file_exists(@upload, :preview)
assert_file_exists(@upload, :original)
end
end
context "for pixiv ugoira" do
setup do
skip "Pixiv credentials not configured" unless Sources::Strategies::Pixiv.enabled?
@source = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364"
end
should "download the file" do
skip unless MediaFile::Ugoira.videos_enabled?
@service = UploadService::Preprocessor.new(source: @source)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(2804, @upload.file_size)
assert_equal("zip", @upload.file_ext)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @upload.md5)
assert_file_exists(@upload, :sample)
assert_file_exists(@upload, :original)
end
end
context "for null" do
setup do
@source = "https://cdn.donmai.us/original/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
end
should "download the file" do
@service = UploadService::Preprocessor.new(source: @source)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(181309, @upload.file_size)
assert_equal("jpg", @upload.file_ext)
assert_equal("93f4dd66ef1eb11a89e56d31f9adc8d0", @upload.md5)
assert_file_exists(@upload, :preview)
assert_file_exists(@upload, :sample)
assert_file_exists(@upload, :original)
end
end
context "for a video" do
setup do
@source = "https://cdn.donmai.us/original/b7/cb/b7cb80092be273771510952812380fa2.mp4"
end
should "work for a video" do
@service = UploadService::Preprocessor.new(source: @source)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_not_nil(@upload.md5)
assert_equal("mp4", @upload.file_ext)
assert_operator(@upload.file_size, :>, 0)
assert_not_nil(@upload.source)
assert_file_exists(@upload, :preview)
assert_file_exists(@upload, :original)
end
end
context "on timeout errors" do
setup do
@source = "https://cdn.donmai.us/original/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
Danbooru::Http.any_instance.stubs(:get).raises(HTTP::TimeoutError)
end
should "leave the upload in an error state" do
@service = UploadService::Preprocessor.new(source: @source)
@upload = @service.start!
assert_match(/error:/, @upload.status)
end
end
context "for an invalid content type" do
should "fail" do
upload = UploadService::Preprocessor.new(source: "http://www.example.com").start!
assert_match(/\Aerror:.*File ext is invalid/, upload.status)
end
end
end
context "#finish!" do
setup do
skip "Twitter credentials not configured" unless Sources::Strategies::Twitter.enabled?
CurrentUser.user = travel_to(1.month.ago) do
FactoryBot.create(:user)
end
CurrentUser.ip_addr = "127.0.0.1"
@source = "https://twitter.com/nounproject/status/540944400767922176"
end
should "overwrite the attributes" do
@service = UploadService::Preprocessor.new(source: @source, rating: 'e')
@upload = @service.start!
@service.finish!
@upload.reload
assert_equal('e', @upload.rating)
end
end
end
context "::Replacer" do
context "for a file replacement" do
setup do
@@ -518,63 +311,12 @@ class UploadServiceTest < ActiveSupport::TestCase
CurrentUser.ip_addr = nil
end
context "automatic tagging" do
should "tag animated png files" do
upload = UploadService.new(file: upload_file("test/files/apng/normal_apng.png")).start!
assert_match(/animated_png/, upload.post.tag_string)
end
should "tag animated gif files" do
upload = UploadService.new(file: upload_file("test/files/test-animated-86x52.gif")).start!
assert_match(/animated_gif/, upload.post.tag_string)
end
should "not tag static gif files" do
upload = UploadService.new(file: upload_file("test/files/test-static-32x32.gif")).start!
assert_no_match(/animated_gif/, upload.post.tag_string)
end
end
context "that is too large" do
setup do
Danbooru.config.stubs(:max_image_resolution).returns(31 * 31)
end
should "should fail validation" do
service = subject.new(file: upload_file("test/files/test-large.jpg"))
upload = service.start!
assert_match(/image resolution is too large/, upload.status)
end
end
context "with a preprocessing predecessor" do
setup do
@predecessor = FactoryBot.create(:source_upload, status: "preprocessing", source: @source, image_height: 0, image_width: 0, file_ext: "jpg")
end
should "schedule a job later" do
service = subject.new(source: @source)
predecessor = service.start!
assert_enqueued_jobs(1, only: UploadServiceDelayedStartJob)
assert_equal(@predecessor, predecessor)
end
end
context "with a preprocessed predecessor" do
setup do
@predecessor = FactoryBot.create(:source_upload, status: "preprocessed", source: @source, image_height: 0, image_width: 0, file_size: 1, md5: 'd34e4cf0a437a5d65f8e82b7bcd02606', file_ext: "jpg")
@tags = 'hello world'
end
should "update the predecessor" do
service = subject.new(source: @source, tag_string: @tags)
predecessor = service.start!
assert_equal(@predecessor, predecessor)
assert_equal(@tags, predecessor.tag_string.strip)
end
context "when the file has already been uploaded" do
setup do
@asset = MediaAsset.find_by_md5("d34e4cf0a437a5d65f8e82b7bcd02606")
@@ -590,39 +332,7 @@ class UploadServiceTest < ActiveSupport::TestCase
end
end
context "with no predecessor" do
should "create an upload" do
service = subject.new(source: @source)
assert_difference(-> { Upload.count }) do
service.start!
end
end
should "assign the rating from tags" do
service = subject.new(source: @source, tag_string: "rating:safe blah")
upload = service.start!
assert_equal(true, upload.valid?)
assert_equal("s", upload.rating)
assert_equal("rating:safe blah", upload.tag_string)
assert_equal("s", upload.post.rating)
assert_equal("blah", upload.post.tag_string)
end
end
context "with a source containing unicode characters" do
should "upload successfully" do
source1 = "https://cdn.donmai.us/original/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg?one=東方&two=a%20b"
source2 = "https://cdn.donmai.us/original/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg?one=%E6%9D%B1%E6%96%B9&two=a%20b"
service = subject.new(source: source1, rating: "s")
assert_nothing_raised { @upload = service.start! }
assert_equal(true, @upload.is_completed?)
assert_equal(source2, @upload.source)
end
should "normalize unicode characters in the source field" do
source1 = "poke\u0301mon" # pokémon (nfd form)
source2 = "pok\u00e9mon" # pokémon (nfc form)
@@ -632,51 +342,6 @@ class UploadServiceTest < ActiveSupport::TestCase
assert_equal(source2, @upload.source)
end
end
context "without a file or a source url" do
should "fail gracefully" do
service = subject.new(source: "blah", rating: "s")
assert_nothing_raised { @upload = service.start! }
assert_equal(true, @upload.is_errored?)
assert_match(/No file or source URL provided/, @upload.status)
end
end
context "with both a file and a source url" do
should "upload the file and set the source field to the given source" do
service = subject.new(file: upload_file("test/files/test.jpg"), source: "http://www.example.com", rating: "s")
assert_nothing_raised { @upload = service.start! }
assert_equal(true, @upload.is_completed?)
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
assert_equal("http://www.example.com", @upload.source)
end
end
context "for a corrupted image" do
should "fail for a corrupted jpeg" do
@bad_jpeg_path = "test/files/test-corrupt.jpg"
upload = upload_from_file(@bad_jpeg_path)
assert_match(/corrupt/, upload.status)
end
should "fail for a corrupted gif" do
@bad_gif_path = "test/files/test-corrupt.gif"
upload = upload_from_file(@bad_gif_path)
assert_match(/corrupt/, upload.status)
end
# https://schaik.com/pngsuite/pngsuite_xxx_png.html
should "fail for a corrupted png" do
@bad_png_path = "test/files/test-corrupt.png"
upload = upload_from_file(@bad_png_path)
assert_match(/corrupt/, upload.status)
end
end
end
context "#create_post_from_upload" do
@@ -767,31 +432,4 @@ class UploadServiceTest < ActiveSupport::TestCase
end
end
end
context "Upload#prune!" do
setup do
@user = create(:user, created_at: 1.year.ago)
end
should "delete stale upload records" do
@upload = as(@user) { UploadService.new(file: upload_file("test/files/test.jpg")).start! }
@upload.update!(created_at: 1.month.ago)
assert_difference("Upload.count", -1) { Upload.prune! }
end
should "work on uploads without a file" do
@upload = as(@user) { UploadService.new(source: "http://14903gf0vm3g134yjq3n535yn3n.com/does_not_exist.jpg").start! }
assert(@upload.is_errored?)
assert_difference("Upload.count", -1) { @upload.destroy! }
end
should "work on uploads with an invalid file" do
@upload = as(@user) { UploadService.new(file: upload_file("test/files/test-empty.bin")).start! }
assert(@upload.is_errored?)
assert_difference("Upload.count", -1) { @upload.destroy! }
end
end
end