uploads: allow uploading .zip, .rar., and .7z files from disk.

Allow uploading .zip, .rar, and .7z files from disk. The archive will be extracted and the images
inside will be uploaded.

This only works for archive files uploaded from disk, not from a source URL.

Post source URLs will look something like this: "file://foo.zip/1.jpg", "file://foo.zip/2.jpg", etc.
Sometimes artists uses Shift JIS or other encodings instead of UTF-8 for filenames. In these cases
we just assume the filename is UTF-8 and replace invalid characters with '?', so filenames might be
wrong in some cases.

There are various protections to prevent uploading malicious archive files:

* Archives with more than 100 files aren't allowed.
* Archives that decompress to more than 100MB aren't allowed.
* Archives with filenames containing '..' components aren't allowed (e.g. '../../../../../etc/passwd').
* Archives with filenames containing absolute paths aren't allowed (e.g. '/etc/passwd').
* Archives containing symlinks aren't allowed (e.g. 'foo -> /etc/passwd').
* Archive types other than .zip, .rar, and .7z aren't allowed (e.g. .tar.gz, .cpio).
* File permissions, owners, and other metadata are ignored.

Partial fix for #5340: Add support for extracting archive attachments from certain sources

test/functional/uploads_controller_test.rb

@@ -134,7 +134,7 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
 
       should "fail if given both a file and source" do
         assert_no_difference("Upload.count") do
-          file = File.open("test/files/test.jpg")
+          file = Rack::Test::UploadedFile.new("test/files/test.jpg")
           source = "https://files.catbox.moe/om3tcw.webm"
           post_auth uploads_path(format: :json), @user, params: { upload: { files: { "0" => file }, source: source }}
         end
@@ -296,6 +296,48 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
         end
       end
 
+      context "for an unsupported archive type" do
+        should "fail for a .tar file" do
+          create_upload!("test/files/archive/ugoira.tar", user: @user)
+          assert_match("File is not an image or video", Upload.last.error)
+        end
+
+        should "fail for a .tar.gz file" do
+          create_upload!("test/files/archive/ugoira.tar.gz", user: @user)
+          assert_match("File is not an image or video", Upload.last.error)
+        end
+
+        should "fail for an archive containing more than 100 files" do
+          create_upload!("test/files/archive/bomb-10k-files.7z", user: @user)
+          assert_response 422
+          assert_match("'bomb-10k-files.7z' contains too many files (max 100 files per upload)", response.parsed_body.dig("errors", "base", 0))
+        end
+
+        should "fail for a decompression bomb" do
+          create_upload!("test/files/archive/bomb-1-1G.rar", user: @user)
+          assert_response 422
+          assert_match("'bomb-1-1G.rar' is too large (uncompressed size: 1,000 MB; max size: 100 MB)", response.parsed_body.dig("errors", "base", 0))
+        end
+
+        should "fail for an archive containing absolute paths" do
+          create_upload!("test/files/archive/absolute-path.7z", user: @user)
+          assert_response 422
+          assert_match("'/tmp/foo/foo.txt' in 'absolute-path.7z' can't start with '/'", response.parsed_body.dig("errors", "base", 0))
+        end
+
+        should "fail for an archive containing '..' paths" do
+          create_upload!("test/files/archive/zip-slip.zip", user: @user)
+          assert_response 422
+          assert_match(/'.*' in 'zip-slip\.zip' can't contain '\.\.' components/, response.parsed_body.dig("errors", "base", 0))
+        end
+
+        should "fail for an archive containing symlinks" do
+          create_upload!("test/files/archive/symlink.zip", user: @user)
+          assert_response 422
+          assert_match("'passwd' in 'symlink.zip' isn't a regular file", response.parsed_body.dig("errors", "base", 0))
+        end
+      end
+
       context "when re-uploading a media asset stuck in the 'processing' state" do
         should "mark the asset as failed" do
           asset = create(:media_asset, file: File.open("test/files/test.jpg"), status: "processing")
@@ -395,6 +437,36 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
         should_upload_successfully("test/files/webp/lossy_alpha1.webp")
       end
 
+      context "uploading a .zip file from your computer" do
+        should "work" do
+          upload = assert_successful_upload("test/files/archive/ugoira.zip", user: @user)
+
+          assert_equal(5, upload.media_asset_count)
+          assert_equal(5, upload.upload_media_assets.size)
+          assert_equal("file://ugoira.zip/000000.jpg", upload.upload_media_assets[0].source_url)
+        end
+      end
+
+      context "uploading a .rar file from your computer" do
+        should "work" do
+          upload = assert_successful_upload("test/files/archive/ugoira.rar", user: @user)
+
+          assert_equal(5, upload.media_asset_count)
+          assert_equal(5, upload.upload_media_assets.size)
+          assert_equal("file://ugoira.rar/000000.jpg", upload.upload_media_assets[0].source_url)
+        end
+      end
+
+      context "uploading a .7z file from your computer" do
+        should "work" do
+          upload = assert_successful_upload("test/files/archive/ugoira.7z", user: @user)
+
+          assert_equal(5, upload.media_asset_count)
+          assert_equal(5, upload.upload_media_assets.size)
+          assert_equal("file://ugoira.7z/000000.jpg", upload.upload_media_assets[0].source_url)
+        end
+      end
+
       context "uploading multiple files from your computer" do
         should "work" do
           files = {

test/unit/danbooru_archive_test.rb

@@ -117,7 +117,8 @@ class DanbooruArchiveTest < ActiveSupport::TestCase
         assert_equal("ZIP 2.0 (uncompressed)", Danbooru::Archive.open("test/files/archive/ugoira.zip").format)
         assert_equal("RAR5", Danbooru::Archive.open("test/files/archive/ugoira.rar").format)
         assert_equal("7-Zip", Danbooru::Archive.open("test/files/archive/ugoira.7z").format)
-        assert_equal("7-Zip", Danbooru::Archive.open("test/files/archive/ugoira.tar.7z").format)
+        assert_equal("GNU tar format", Danbooru::Archive.open("test/files/archive/ugoira.tar").format)
+        assert_equal("GNU tar format", Danbooru::Archive.open("test/files/archive/ugoira.tar.gz").format)
       end
     end
 
@@ -126,7 +127,8 @@ class DanbooruArchiveTest < ActiveSupport::TestCase
         assert_equal(:zip, Danbooru::Archive.open("test/files/archive/ugoira.zip").file_ext)
         assert_equal(:rar, Danbooru::Archive.open("test/files/archive/ugoira.rar").file_ext)
         assert_equal(:"7z", Danbooru::Archive.open("test/files/archive/ugoira.7z").file_ext)
-        assert_equal(:"7z", Danbooru::Archive.open("test/files/archive/ugoira.tar.7z").file_ext)
+        assert_equal(:bin, Danbooru::Archive.open("test/files/archive/ugoira.tar").file_ext)
+        assert_equal(:bin, Danbooru::Archive.open("test/files/archive/ugoira.tar.gz").file_ext)
       end
     end
 
@@ -139,5 +141,40 @@ class DanbooruArchiveTest < ActiveSupport::TestCase
         assert_match(/^-rw-rw-r-- *0 0 *1639 2014-10-05 23:31:06 000000\.jpg$/, output.tap(&:rewind).read)
       end
     end
+
+    should "detect directory traversal attacks" do
+      archive = Danbooru::Archive.open!("test/files/archive/zip-slip.zip")
+
+      assert_equal(true, archive.entries.any? { |e| e.directory_traversal? })
+      assert_raises(Danbooru::Archive::Error) { archive.extract! }
+    end
+
+    should "detect symlinks" do
+      archive = Danbooru::Archive.open!("test/files/archive/symlink.zip")
+
+      assert_equal(true, archive.entries.any? { |e| !e.file? && !e.directory? })
+      assert_raises(Danbooru::Archive::Error) { archive.extract! }
+    end
+
+    should "detect absolute paths" do
+      archive = Danbooru::Archive.open!("test/files/archive/absolute-path.7z")
+
+      assert_equal(true, archive.entries.any? { |e| e.pathname.starts_with?("/") })
+      assert_raises(Danbooru::Archive::Error) { archive.extract! }
+    end
+
+    should "detect archives with large numbers of files" do
+      archive = Danbooru::Archive.open!("test/files/archive/bomb-10k-files.7z")
+
+      assert_equal(true, archive.exists? { |_, count| count > 100 })
+      assert_equal(10_000, archive.file_count)
+    end
+
+    should "detect decompression bombs" do
+      archive = Danbooru::Archive.open!("test/files/archive/bomb-1-1G.rar")
+
+      assert_equal(1, archive.file_count)
+      assert_equal(1_048_576_000, archive.uncompressed_size)
+    end
   end
 end