jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

uploads: allow uploading .zip, .rar., and .7z files from disk.

Browse Source 
Allow uploading .zip, .rar, and .7z files from disk. The archive will be extracted and the images
inside will be uploaded.

This only works for archive files uploaded from disk, not from a source URL.

Post source URLs will look something like this: "file://foo.zip/1.jpg", "file://foo.zip/2.jpg", etc.
Sometimes artists uses Shift JIS or other encodings instead of UTF-8 for filenames. In these cases
we just assume the filename is UTF-8 and replace invalid characters with '?', so filenames might be
wrong in some cases.

There are various protections to prevent uploading malicious archive files:

* Archives with more than 100 files aren't allowed.
* Archives that decompress to more than 100MB aren't allowed.
* Archives with filenames containing '..' components aren't allowed (e.g. '../../../../../etc/passwd').
* Archives with filenames containing absolute paths aren't allowed (e.g. '/etc/passwd').
* Archives containing symlinks aren't allowed (e.g. 'foo -> /etc/passwd').
* Archive types other than .zip, .rar, and .7z aren't allowed (e.g. .tar.gz, .cpio).
* File permissions, owners, and other metadata are ignored.

Partial fix for #5340: Add support for extracting archive attachments from certain sources
This commit is contained in:
evazion
2022-11-16 01:53:50 -06:00
parent d791924aad
commit 2deae38a4e
14 changed files with 232 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
app/models/upload.rb
View File

@@ -4,6 +4,10 @@ class Upload < ApplicationRecord
extend Memoist
class Error < StandardError; end
# The list of allowed archive file types.
ARCHIVE_FILE_TYPES = %i[zip rar 7z]
# The maximum number of files allowed per upload.
MAX_FILES_PER_UPLOAD = 100
# The maximum number of 'pending' or 'processing' media assets a single user can have at once.
@@ -18,11 +22,11 @@ class Upload < ApplicationRecord
normalize :source, :normalize_source
validates :files, length: { maximum: MAX_FILES_PER_UPLOAD, message: "can't have more than #{MAX_FILES_PER_UPLOAD} files per upload" }
validates :source, format: { with: %r{\Ahttps?://}i, message: "is not a valid URL" }, if: -> { source.present? }
validates :referer_url, format: { with: %r{\Ahttps?://}i, message: "is not a valid URL" }, if: -> { referer_url.present? }
validate :validate_file_and_source, on: :create
validate :uploader_is_not_limited, on: :create
validate :validate_archive_files, on: :create
validate :validate_uploader_is_not_limited, on: :create
after_create :async_process_upload!
@@ -69,13 +73,40 @@ class Upload < ApplicationRecord
end
end
def uploader_is_not_limited
def validate_uploader_is_not_limited
queued_asset_count = uploader.upload_media_assets.unfinished.count
if queued_asset_count > MAX_QUEUED_ASSETS
errors.add(:base, "You have too many images queued for upload (queued: #{queued_asset_count}; limit: #{MAX_QUEUED_ASSETS}). Try again later.")
end
end
def validate_archive_files
return unless files.present?
archive_files.each do |archive, filename|
if !archive.file_ext.in?(ARCHIVE_FILE_TYPES)
errors.add(:base, "'#{filename}' is not a supported file type")
elsif archive.exists? { |_, count| count > MAX_FILES_PER_UPLOAD }
# XXX Potential zip bomb containing thousands of files; don't process it any further.
errors.add(:base, "'#{filename}' contains too many files (max #{MAX_FILES_PER_UPLOAD} files per upload)")
next
elsif archive.uncompressed_size > MediaAsset::MAX_FILE_SIZE
errors.add(:base, "'#{filename}' is too large (uncompressed size: #{archive.uncompressed_size.to_fs(:human_size)}; max size: #{MediaAsset::MAX_FILE_SIZE.to_fs(:human_size)})")
elsif entry = archive.entries.find { |entry| entry.pathname.starts_with?("/") }
errors.add(:base, "'#{entry.pathname_utf8}' in '#{filename}' can't start with '/'")
elsif entry = archive.entries.find { |entry| entry.directory_traversal? }
errors.add(:base, "'#{entry.pathname_utf8}' in '#{filename}' can't contain '..' components")
elsif entry = archive.entries.find { |entry| !entry.file? && !entry.directory? }
errors.add(:base, "'#{entry.pathname_utf8}' in '#{filename}' isn't a regular file")
end
end
total_files = archive_files.map(&:first).sum(&:file_count) + (files.size - archive_files.size)
if total_files > MAX_FILES_PER_UPLOAD
errors.add(:base, "Can't upload more than #{MAX_FILES_PER_UPLOAD} files at a time (total: #{total_files})")
end
end
end
concerning :SourceMethods do
@@ -132,30 +163,75 @@ class Upload < ApplicationRecord
update!(status: "processing")
if files.present?
upload_media_assets = files.map do |_index, file|
UploadMediaAsset.new(upload: self, file: file.tempfile, source_url: "file://#{file.original_filename}")
end
process_file_upload!
elsif source.present?
page_url = source_extractor.page_url
image_urls = source_extractor.image_urls
if image_urls.empty?
raise Error, "#{source} doesn't contain any images"
end
upload_media_assets = image_urls.map do |image_url|
UploadMediaAsset.new(upload: self, source_url: image_url, page_url: page_url, media_asset: nil)
end
process_source_upload!
else
raise Error, "No file or source given" # Should never happen
end
rescue Exception => e
update!(status: "error", error: e.message)
end
def process_source_upload!
page_url = source_extractor.page_url
image_urls = source_extractor.image_urls
if image_urls.empty?
raise Error, "#{source} doesn't contain any images"
end
upload_media_assets = image_urls.map do |image_url|
UploadMediaAsset.new(upload: self, source_url: image_url, page_url: page_url, media_asset: nil)
end
transaction do
update!(media_asset_count: upload_media_assets.size)
upload_media_assets.each(&:save!)
end
rescue Exception => e
update!(status: "error", error: e.message)
end
def process_file_upload!
tmpdirs = []
upload_media_assets = uploaded_files.flat_map do |file, original_filename|
if file.is_a?(Danbooru::Archive)
tmpdir, filenames = file.extract!
tmpdirs << tmpdir
filenames.map do |filename|
name = "file://#{original_filename}/#{Pathname.new(filename).relative_path_from(tmpdir)}" # "file://foo.zip/foo/1.jpg"
UploadMediaAsset.new(upload: self, file: filename, source_url: name)
end
else
UploadMediaAsset.new(upload: self, file: file, source_url: "file://#{original_filename}")
end
end
transaction do
update!(media_asset_count: upload_media_assets.size)
upload_media_assets.each(&:save!)
end
ensure
tmpdirs.each { |tmpdir| FileUtils.rm_rf(tmpdir) }
end
# The list of files uploaded from disk, with their filenames.
def uploaded_files
files.map do |_index, file|
if FileTypeDetector.new(file.tempfile).file_ext.in?(ARCHIVE_FILE_TYPES)
[Danbooru::Archive.open!(file.tempfile), file.original_filename]
else
[MediaFile.open(file.tempfile), file.original_filename]
end
end
end
# The list of archive files uploaded from disk, with their filenames.
def archive_files
uploaded_files.select do |file, original_filename|
file.is_a?(Danbooru::Archive)
end
end
def source_extractor
@@ -167,5 +243,5 @@ class Upload < ApplicationRecord
[:uploader, :upload_media_assets, :media_assets, :posts]
end
memoize :source_extractor
memoize :source_extractor, :archive_files, :uploaded_files
end