uploads: allow uploading .zip, .rar., and .7z files from disk.

Allow uploading .zip, .rar, and .7z files from disk. The archive will be extracted and the images
inside will be uploaded.

This only works for archive files uploaded from disk, not from a source URL.

Post source URLs will look something like this: "file://foo.zip/1.jpg", "file://foo.zip/2.jpg", etc.
Sometimes artists uses Shift JIS or other encodings instead of UTF-8 for filenames. In these cases
we just assume the filename is UTF-8 and replace invalid characters with '?', so filenames might be
wrong in some cases.

There are various protections to prevent uploading malicious archive files:

* Archives with more than 100 files aren't allowed.
* Archives that decompress to more than 100MB aren't allowed.
* Archives with filenames containing '..' components aren't allowed (e.g. '../../../../../etc/passwd').
* Archives with filenames containing absolute paths aren't allowed (e.g. '/etc/passwd').
* Archives containing symlinks aren't allowed (e.g. 'foo -> /etc/passwd').
* Archive types other than .zip, .rar, and .7z aren't allowed (e.g. .tar.gz, .cpio).
* File permissions, owners, and other metadata are ignored.

Partial fix for #5340: Add support for extracting archive attachments from certain sources
This commit is contained in:
evazion
2022-11-16 01:53:50 -06:00
parent d791924aad
commit 2deae38a4e
14 changed files with 232 additions and 31 deletions

View File

@@ -134,7 +134,7 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
should "fail if given both a file and source" do
assert_no_difference("Upload.count") do
file = File.open("test/files/test.jpg")
file = Rack::Test::UploadedFile.new("test/files/test.jpg")
source = "https://files.catbox.moe/om3tcw.webm"
post_auth uploads_path(format: :json), @user, params: { upload: { files: { "0" => file }, source: source }}
end
@@ -296,6 +296,48 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
end
end
context "for an unsupported archive type" do
should "fail for a .tar file" do
create_upload!("test/files/archive/ugoira.tar", user: @user)
assert_match("File is not an image or video", Upload.last.error)
end
should "fail for a .tar.gz file" do
create_upload!("test/files/archive/ugoira.tar.gz", user: @user)
assert_match("File is not an image or video", Upload.last.error)
end
should "fail for an archive containing more than 100 files" do
create_upload!("test/files/archive/bomb-10k-files.7z", user: @user)
assert_response 422
assert_match("'bomb-10k-files.7z' contains too many files (max 100 files per upload)", response.parsed_body.dig("errors", "base", 0))
end
should "fail for a decompression bomb" do
create_upload!("test/files/archive/bomb-1-1G.rar", user: @user)
assert_response 422
assert_match("'bomb-1-1G.rar' is too large (uncompressed size: 1,000 MB; max size: 100 MB)", response.parsed_body.dig("errors", "base", 0))
end
should "fail for an archive containing absolute paths" do
create_upload!("test/files/archive/absolute-path.7z", user: @user)
assert_response 422
assert_match("'/tmp/foo/foo.txt' in 'absolute-path.7z' can't start with '/'", response.parsed_body.dig("errors", "base", 0))
end
should "fail for an archive containing '..' paths" do
create_upload!("test/files/archive/zip-slip.zip", user: @user)
assert_response 422
assert_match(/'.*' in 'zip-slip\.zip' can't contain '\.\.' components/, response.parsed_body.dig("errors", "base", 0))
end
should "fail for an archive containing symlinks" do
create_upload!("test/files/archive/symlink.zip", user: @user)
assert_response 422
assert_match("'passwd' in 'symlink.zip' isn't a regular file", response.parsed_body.dig("errors", "base", 0))
end
end
context "when re-uploading a media asset stuck in the 'processing' state" do
should "mark the asset as failed" do
asset = create(:media_asset, file: File.open("test/files/test.jpg"), status: "processing")
@@ -395,6 +437,36 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
should_upload_successfully("test/files/webp/lossy_alpha1.webp")
end
context "uploading a .zip file from your computer" do
should "work" do
upload = assert_successful_upload("test/files/archive/ugoira.zip", user: @user)
assert_equal(5, upload.media_asset_count)
assert_equal(5, upload.upload_media_assets.size)
assert_equal("file://ugoira.zip/000000.jpg", upload.upload_media_assets[0].source_url)
end
end
context "uploading a .rar file from your computer" do
should "work" do
upload = assert_successful_upload("test/files/archive/ugoira.rar", user: @user)
assert_equal(5, upload.media_asset_count)
assert_equal(5, upload.upload_media_assets.size)
assert_equal("file://ugoira.rar/000000.jpg", upload.upload_media_assets[0].source_url)
end
end
context "uploading a .7z file from your computer" do
should "work" do
upload = assert_successful_upload("test/files/archive/ugoira.7z", user: @user)
assert_equal(5, upload.media_asset_count)
assert_equal(5, upload.upload_media_assets.size)
assert_equal("file://ugoira.7z/000000.jpg", upload.upload_media_assets[0].source_url)
end
end
context "uploading multiple files from your computer" do
should "work" do
files = {