uploads: fix corrupted image detection.

* Fix corrupted image detection. We were shelling out to vips and trying
  to grep for error messages, but the error message for jpeg files changed.
  Now we load the file in ruby vips, which raises an error on failure.

* Don't attempt to redownload corrupted images. If a download completes
  without any errors yet the downloaded file is corrupt, then something is
  wrong at the source and redownloading is unlikely to help. Let the
  upload fail and the user retry if necessary.

* Validate that all uploads are uncorrupted, including files uploaded
  from a computer, not just files uploaded from a source.
This commit is contained in:
evazion
2020-04-13 14:05:21 -05:00
parent cc31eeafa4
commit dc6575dc76
7 changed files with 48 additions and 56 deletions

View File

@@ -35,19 +35,4 @@ module DanbooruImageResizer
output_file
end
def validate_shell(file)
temp = Tempfile.new("validate")
output, status = Open3.capture2e("vips stats #{file.path} #{temp.path}.v")
# png | jpeg | gif
if output =~ /Read Error|Premature end of JPEG file|Failed to read from given file/m
return false
end
return true
ensure
temp.close
temp.unlink
end
end

View File

@@ -2,8 +2,6 @@ class UploadService
module Utils
module_function
class CorruptFileError < RuntimeError; end
def file_header_to_file_ext(file)
case File.read(file.path, 16)
when /^\xff\xd8/n
@@ -25,6 +23,14 @@ class UploadService
end
end
def corrupt?(filename)
image = Vips::Image.new_from_file(filename, fail: true)
image.stats
false
rescue Vips::Error
true
end
def calculate_ugoira_dimensions(source_path)
folder = Zip::File.new(source_path)
Tempfile.open("ugoira-dim-") do |tempfile|
@@ -180,23 +186,8 @@ class UploadService
return file if file.present?
raise "No file or source URL provided" if upload.source_url.blank?
attempts = 0
begin
download = Downloads::File.new(upload.source_url, upload.referer_url)
file, strategy = download.download!
if !DanbooruImageResizer.validate_shell(file)
raise CorruptFileError.new("File is corrupted")
end
rescue StandardError
if attempts == 3
raise
end
attempts += 1
retry
end
download = Downloads::File.new(upload.source_url, upload.referer_url)
file, strategy = download.download!
if download.data[:ugoira_frame_data].present?
upload.context = {

View File

@@ -6,6 +6,7 @@ class Upload < ApplicationRecord
class FileValidator < ActiveModel::Validator
def validate(record)
validate_file_ext(record)
validate_integrity(record)
validate_md5_uniqueness(record)
validate_video_duration(record)
validate_resolution(record)
@@ -17,6 +18,12 @@ class Upload < ApplicationRecord
end
end
def validate_integrity(record)
if record.file_ext.in?(["jpg", "gif", "png"]) && UploadService::Utils.corrupt?(record.file.path)
record.errors[:file] << "File is corrupted"
end
end
def validate_md5_uniqueness(record)
if record.md5.nil?
return