media assets: add rake task to validate thumbnails.

Update `bin/rails danbooru:images:validate` to work on media assets
instead of posts, and to check that all thumbnails exist.
This commit is contained in:
evazion
2021-12-07 17:59:08 -06:00
parent 8669edd93f
commit 08bcd51ac8

View File

@@ -25,42 +25,50 @@ namespace :danbooru do
end end
end end
# Usage: TAGS="touhou" bin/rails danbooru:images:validate # Usage: bin/rails danbooru:images:validate
# #
# Check whether any images are missing, corrupt, or don't match the # Check whether any images are missing, corrupt, or don't match the
# width/height/size/ext metadata in the database. # width/height/size/ext metadata in the database.
namespace :images do namespace :images do
task validate: :environment do task validate: :environment do
processes = ENV.fetch("PROCESSES", Etc.nprocessors).to_i processes = ENV.fetch("PROCESSES", Etc.nprocessors).to_i
posts = Post.system_tag_match(ENV["TAGS"]).reorder(nil)
posts.parallel_each(in_processes: processes) do |post| MediaAsset.active.parallel_each(in_processes: processes) do |asset|
media_file = MediaFile.open(post.file(:original)) media_file = asset.variant(:original).open_file
raise if post.md5 != media_file.md5 raise if asset.md5 != media_file.md5
raise if post.image_width != media_file.width raise if asset.image_width != media_file.width
raise if post.image_height != media_file.height raise if asset.image_height != media_file.height
raise if post.file_size != media_file.file_size raise if asset.file_size != media_file.file_size
raise if post.file_ext != media_file.file_ext.to_s raise if asset.file_ext != media_file.file_ext.to_s
puts "post ##{post.id}" asset.variants.each do |variant|
f = variant.open_file
raise if f.is_corrupt?
f.close
end
puts { id: asset.id, md5: asset.md5 }.to_json
rescue RuntimeError => e rescue RuntimeError => e
hash = { hash = {
post: { asset: {
id: post.id, id: asset.id,
md5: post.md5, md5: asset.md5,
width: post.image_width, width: asset.image_width,
height: post.image_height, height: asset.image_height,
size: post.file_size, size: asset.file_size,
ext: post.file_ext, ext: asset.file_ext,
}, },
media_file: media_file.as_json.except("metadata"), media_file: media_file.as_json.except("metadata"),
error: e.to_s, error: e.to_s,
} }
STDERR.puts hash.to_json STDERR.puts hash.to_json
rescue StandardError => e rescue StandardError => e
hash = { id: post.id, md5: post.md5, error: e.to_s } hash = { id: asset.id, error: e.to_s }
STDERR.puts hash.to_json STDERR.puts hash.to_json
ensure
media_file&.close
end end
end end