uploads: add support for uploading .avif files.

Features of AVIF include:

* Lossless and lossy compression.
* High dynamic range (HDR) images
* Wide color gamut images (i.e. 10- and 12-bit color depths)
* Transparency (through alpha planes).
* Animations (with an optional cover image).
* Auxiliary image sequences, where the file contains a single primary
  image and a short secondary video, like Apple's Live Photos.
* Metadata rotation, mirroring, and cropping.

The AVIF format is still relatively new and some of these features aren't well
supported by browsers or other software:

* Animated AVIFs aren't supported by Firefox or by libvips.
* HDR images aren't supported by Firefox.
* Rotated, mirrored, and cropped AVIFs aren't supported by Firefox or Chrome.
* Image grids, where the file contains multiple images that are tiled
  together into one big image, aren't supported by Firefox.
* AVIF as a whole has only been supported for a year or two by Chrome
  and Firefox, and less than a year by Safari.

For these reasons, only basic AVIFs that don't use animation, rotation,
cropping, or image grids can be uploaded.

app/logical/exif_tool.rb

@@ -48,7 +48,7 @@ class ExifTool
     end
 
     def is_animated?
-      frame_count.to_i > 1
+      frame_count.to_i > 1 || is_animated_avif?
     end
 
     def is_animated_gif?
@@ -59,6 +59,10 @@ class ExifTool
       file_ext == :png && is_animated?
     end
 
+    def is_animated_avif?
+      file_ext == :avif && metadata["QuickTime:CompatibleBrands"].to_a.include?("avis")
+    end
+
     # @see https://exiftool.org/TagNames/JPEG.html
     # @see https://exiftool.org/TagNames/PNG.html
     # @see https://danbooru.donmai.us/posts?tags=exif:File:ColorComponents=1
@@ -66,12 +70,33 @@ class ExifTool
     def is_greyscale?
       metadata["File:ColorComponents"] == 1 ||
         metadata["PNG:ColorType"] == "Grayscale" ||
-        metadata["PNG:ColorType"] == "Grayscale with Alpha"
+        metadata["PNG:ColorType"] == "Grayscale with Alpha" ||
+        metadata["QuickTime:ChromaFormat"].to_s.match?(/Monochrome/) # "Monochrome 4:0:0"
     end
 
     # https://exiftool.org/TagNames/EXIF.html
     def is_rotated?
-      file_ext == :jpg && metadata["IFD0:Orientation"].in?(["Rotate 90 CW", "Rotate 270 CW", "Rotate 180"])
+      case file_ext
+      when :jpg
+        metadata["IFD0:Orientation"].in?(["Rotate 90 CW", "Rotate 270 CW", "Rotate 180"])
+      when :avif
+        metadata["QuickTime:Rotation"].present?
+      else
+        false
+      end
+    end
+
+    # AVIF files can be cropped with the "CleanAperture" (aka "clap") tag.
+    def is_cropped?
+      file_ext == :avif && metadata["QuickTime:CleanAperture"].present?
+    end
+
+    # AVIF files can be a collection of smaller images combined in a grid to
+    # form a larger image. This is done to reduce memory usage during encoding.
+    #
+    # https://0xc0000054.github.io/pdn-avif/using-image-grids.html
+    def is_grid_image?
+      file_ext == :avif && metadata["Meta:MetaImageSize"].present?
     end
 
     # Some animations technically have a finite loop count, but loop for hundreds
@@ -124,6 +149,8 @@ class ExifTool
         :png
       elsif has_key?("GIF:GIFVersion")
         :gif
+      elsif metadata["QuickTime:CompatibleBrands"].to_a.include?("avif") || metadata["QuickTime:CompatibleBrands"].to_a.include?("avis")
+        :avif
       elsif has_key?("QuickTime:MovieHeaderVersion")
         :mp4
       elsif has_key?("Matroska:DocType")

app/logical/ffmpeg.rb

@@ -32,13 +32,17 @@ class FFmpeg
   end
 
   # Get file metadata using ffprobe.
+  #
   # @see https://ffmpeg.org/ffprobe.html
   # @see https://gist.github.com/nrk/2286511
-  # @return [Hash] a hash of the file's metadata
+  #
+  # @return [Hash] A hash of the file's metadata. Will be empty if reading the file failed for any reason.
   def metadata
     output = shell!("ffprobe -v quiet -print_format json -show_format -show_streams #{file.path.shellescape}")
     json = JSON.parse(output)
     json.with_indifferent_access
+  rescue Error
+    {}
   end
 
   def width
@@ -64,7 +68,7 @@ class FFmpeg
 
   # @return [Integer, nil] The number of frames in the video or animation, or nil if unknown.
   def frame_count
-    if video_streams.first.has_key?(:nb_frames)
+    if video_streams.first&.has_key?(:nb_frames)
       video_streams.first[:nb_frames].to_i
     elsif playback_info.has_key?(:frame)
       playback_info[:frame].to_i
@@ -80,11 +84,11 @@ class FFmpeg
   end
 
   def video_streams
-    metadata[:streams].select { |stream| stream[:codec_type] == "video" }
+    metadata[:streams].to_a.select { |stream| stream[:codec_type] == "video" }
   end
 
   def audio_streams
-    metadata[:streams].select { |stream| stream[:codec_type] == "audio" }
+    metadata[:streams].to_a.select { |stream| stream[:codec_type] == "audio" }
   end
 
   def has_audio?

app/logical/media_file.rb

@@ -26,7 +26,7 @@ class MediaFile
     file = Kernel.open(file, "r", binmode: true) unless file.respond_to?(:read)
 
     case file_ext(file)
-    when :jpg, :gif, :png
+    when :jpg, :gif, :png, :avif
       MediaFile::Image.new(file, **options)
     when :swf
       MediaFile::Flash.new(file, **options)
@@ -66,6 +66,9 @@ class MediaFile
     # M4V (rare) - Apple iTunes Video (https://en.wikipedia.org/wiki/M4V)
     when /\A....ftyp(?:isom|iso5|3gp5|mp42|avc1|M4V)/
       :mp4
+    # https://aomediacodec.github.io/av1-avif/#brands-overview
+    when /\A....ftyp(?:avif|avis)/
+      :avif
     when /\APK\x03\x04/
       :zip
     else
@@ -129,9 +132,14 @@ class MediaFile
     ExifTool.new(file).metadata
   end
 
+  # @return [Boolean] True if the file is supported by Danbooru. Certain files may be unsupported because they use features we don't support.
+  def is_supported?
+    true
+  end
+
   # @return [Boolean] true if the file is an image
   def is_image?
-    file_ext.in?([:jpg, :png, :gif])
+    file_ext.in?([:jpg, :png, :gif, :avif])
   end
 
   # @return [Boolean] true if the file is a video
@@ -189,7 +197,7 @@ class MediaFile
   # @param width [Integer] the max width of the image
   # @param height [Integer] the max height of the image
   # @param options [Hash] extra options when generating the preview
-  # @return [MediaFile] a preview file
+  # @return [MediaFile, nil] a preview file, or nil if we can't generate a preview for this file type (e.g. Flash files)
   def preview(width, height, **options)
     nil
   end
@@ -216,6 +224,7 @@ class MediaFile
       file_size: file_size,
       md5: md5,
       is_corrupt?: is_corrupt?,
+      is_supported?: is_supported?,
       duration: duration,
       frame_count: frame_count,
       frame_rate: frame_rate,

app/logical/media_file/image.rb

@@ -11,6 +11,16 @@ class MediaFile::Image < MediaFile
     [0, 0]
   end
 
+  def is_supported?
+    case file_ext
+    when :avif
+      # XXX Mirrored AVIFs should be unsupported too, but we currently can't detect the mirrored flag using exiftool or ffprobe.
+      !metadata.is_rotated? && !metadata.is_cropped? && !metadata.is_grid_image? && !metadata.is_animated_avif?
+    else
+      true
+    end
+  end
+
   def is_corrupt?
     image.stats
     false
@@ -28,6 +38,8 @@ class MediaFile::Image < MediaFile
       image.get("n-pages")
     elsif file_ext == :png
       metadata.fetch("PNG:AnimationFrames", 1)
+    elsif file_ext == :avif
+      video.frame_count
     else
       nil
     end
@@ -112,6 +124,10 @@ class MediaFile::Image < MediaFile
     file_ext == :png && is_animated?
   end
 
+  def is_animated_avif?
+    file_ext == :avif && is_animated?
+  end
+
   # Return true if the image has an embedded ICC color profile.
   def has_embedded_profile?
     image.icc_import(embedded: true)

app/logical/user_name_validator.rb

@@ -24,7 +24,7 @@ class UserNameValidator < ActiveModel::EachValidator
       rec.errors.add(attr, "can't start with '#{name.first}'")
     elsif name =~ /[[:punct:]]\z/
       rec.errors.add(attr, "can't end with '#{name.last}'")
-    elsif name =~ /\.(html|json|xml|atom|rss|txt|js|css|csv|png|jpg|jpeg|gif|png|mp4|webm|zip|pdf|exe|sitemap)\z/i
+    elsif name =~ /\.(html|json|xml|atom|rss|txt|js|css|csv|png|jpg|jpeg|gif|png|avif|webp|mp4|webm|zip|pdf|exe|sitemap)\z/i
       rec.errors.add(attr, "can't end with a file extension")
     elsif name =~ /__/
       rec.errors.add(attr, "can't contain multiple underscores in a row")

app/models/media_asset.rb

@@ -3,7 +3,7 @@
 class MediaAsset < ApplicationRecord
   class Error < StandardError; end
 
-  FILE_TYPES = %w[jpg png gif mp4 webm swf zip]
+  FILE_TYPES = %w[jpg png gif avif mp4 webm swf zip]
   FILE_KEY_LENGTH = 9
   VARIANTS = %i[preview 180x180 360x360 720x720 sample original]
   MAX_FILE_SIZE = Danbooru.config.max_file_size.to_i
@@ -273,6 +273,8 @@ class MediaAsset < ApplicationRecord
       def validate_media_file!(media_file, uploader)
         if !media_file.file_ext.to_s.in?(FILE_TYPES)
           raise Error, "File is not an image or video"
+        elsif !media_file.is_supported?
+          raise Error, "File type is not supported"
         elsif media_file.is_corrupt?
           raise Error, "File is corrupt"
         elsif media_file.file_size > MAX_FILE_SIZE
@@ -370,7 +372,7 @@ class MediaAsset < ApplicationRecord
 
   concerning :FileTypeMethods do
     def is_image?
-      file_ext.in?(%w[jpg png gif])
+      file_ext.in?(%w[jpg png gif avif])
     end
 
     def is_static_image?

app/models/post.rb

@@ -188,7 +188,7 @@ class Post < ApplicationRecord
     end
 
     def is_image?
-      file_ext =~ /jpg|gif|png/i
+      file_ext =~ /jpg|gif|png|avif/i
     end
 
     def is_flash?

test/files/avif/README.md

@@ -0,0 +1,7 @@
+Test file sources:
+
+* https://github.com/link-u/avif-sample-images
+* https://github.com/AOMediaCodec/av1-avif/tree/master/testFiles
+* https://github.com/AOMediaCodec/libavif/tree/main/tests/data
+* https://colinbendell.github.io/webperf/animated-gif-decode/avif.html
+* https://0xc0000054.github.io/pdn-avif/using-image-grids.html

test/functional/uploads_controller_test.rb

@@ -195,6 +195,33 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
         end
       end
 
+      context "for an unsupported AVIF file" do
+        should "fail for a grid image" do
+          create_upload!("test/files/avif/Image grid example.avif", user: @user)
+          assert_match("File type is not supported", Upload.last.error)
+        end
+
+        should "fail for a cropped image" do
+          create_upload!("test/files/avif/kimono.crop.avif", user: @user)
+          assert_match("File type is not supported", Upload.last.error)
+        end
+
+        should "fail for a rotated image" do
+          create_upload!("test/files/avif/kimono.rotate90.avif", user: @user)
+          assert_match("File type is not supported", Upload.last.error)
+        end
+
+        should "fail for an image sequence" do
+          create_upload!("test/files/avif/sequence-with-pitm.avif", user: @user)
+          assert_match("File type is not supported", Upload.last.error)
+        end
+
+        should "fail for a still image with an auxiliary image sequence" do
+          create_upload!("test/files/avif/sequence-with-pitm-avif-major.avif", user: @user)
+          assert_match("File type is not supported", Upload.last.error)
+        end
+      end
+
       context "for a video longer than the video length limit" do
         should "fail for a regular user" do
           create_upload!("https://cdn.donmai.us/original/63/cb/63cb09f2526ef3ac14f11c011516ad9b.webm", user: @user)
@@ -267,6 +294,12 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
         should_upload_successfully("test/files/test-512x512.webm")
         should_upload_successfully("test/files/test-audio.m4v")
         # should_upload_successfully("test/files/compressed.swf")
+
+        should_upload_successfully("test/files/avif/fox.profile0.8bpc.yuv420.monochrome.avif")
+        should_upload_successfully("test/files/avif/hdr_cosmos01000_cicp9-16-9_yuv420_limited_qp40.avif")
+        should_upload_successfully("test/files/avif/hdr_cosmos01000_cicp9-16-9_yuv444_full_qp40.avif")
+        should_upload_successfully("test/files/avif/paris_icc_exif_xmp.avif")
+        should_upload_successfully("test/files/avif/tiger_3layer_1res.avif")
       end
 
       context "uploading multiple files from your computer" do

test/unit/media_file_test.rb

@@ -21,6 +21,10 @@ class MediaFileTest < ActiveSupport::TestCase
       assert_equal([32, 32], MediaFile.open("test/files/test-static-32x32.gif").dimensions)
     end
 
+    should "determine the correct dimensions for an AVIF file" do
+      assert_equal([2048, 858], MediaFile.open("test/files/avif/hdr_cosmos01000_cicp9-16-9_yuv420_limited_qp40.avif").dimensions)
+    end
+
     should "determine the correct dimensions for a webm file" do
       skip unless MediaFile.videos_enabled?
       assert_equal([512, 512], MediaFile.open("test/files/test-512x512.webm").dimensions)
@@ -85,6 +89,12 @@ class MediaFileTest < ActiveSupport::TestCase
       assert_equal(:gif, MediaFile.open("test/files/test-static-32x32.gif").file_ext)
     end
 
+    should "determine the correct extension for an AVIF file" do
+      Dir["test/files/avif/*.avif"].each do |file|
+        assert_equal(:avif, MediaFile.open(file).file_ext)
+      end
+    end
+
     should "determine the correct extension for a webm file" do
       assert_equal(:webm, MediaFile.open("test/files/test-512x512.webm").file_ext)
     end
@@ -127,6 +137,7 @@ class MediaFileTest < ActiveSupport::TestCase
       assert_equal([150, 101], MediaFile.open("test/files/test.jpg").preview(150, 150).dimensions)
       assert_equal([113, 150], MediaFile.open("test/files/test.png").preview(150, 150).dimensions)
       assert_equal([150, 150], MediaFile.open("test/files/test.gif").preview(150, 150).dimensions)
+      assert_equal([150, 63], MediaFile.open("test/files/avif/hdr_cosmos01000_cicp9-16-9_yuv420_limited_qp40.avif").preview(150, 150).dimensions)
     end
 
     should "generate a preview image for an animated image" do
@@ -294,6 +305,72 @@ class MediaFileTest < ActiveSupport::TestCase
     end
   end
 
+  context "an AVIF file" do
+    should "be able to read AVIF files" do
+      Dir["test/files/avif/*.avif"].each do |file|
+        assert_nothing_raised { MediaFile.open(file).attributes }
+      end
+    end
+
+    should "detect supported files" do
+      assert_equal(true, MediaFile.open("test/files/avif/paris_icc_exif_xmp.avif").is_supported?)
+      assert_equal(true, MediaFile.open("test/files/avif/hdr_cosmos01000_cicp9-16-9_yuv420_limited_qp40.avif").is_supported?)
+      assert_equal(true, MediaFile.open("test/files/avif/hdr_cosmos01000_cicp9-16-9_yuv444_full_qp40.avif").is_supported?)
+      assert_equal(true, MediaFile.open("test/files/avif/fox.profile0.8bpc.yuv420.monochrome.avif").is_supported?)
+      assert_equal(true, MediaFile.open("test/files/avif/tiger_3layer_1res.avif").is_supported?)
+    end
+
+    should "detect unsupported files" do
+      assert_equal(false, MediaFile.open("test/files/avif/Image grid example.avif").is_supported?)
+      assert_equal(false, MediaFile.open("test/files/avif/kimono.crop.avif").is_supported?)
+      assert_equal(false, MediaFile.open("test/files/avif/kimono.rotate90.avif").is_supported?)
+      assert_equal(false, MediaFile.open("test/files/avif/sequence-with-pitm-avif-major.avif").is_supported?)
+      assert_equal(false, MediaFile.open("test/files/avif/sequence-with-pitm.avif").is_supported?)
+      assert_equal(false, MediaFile.open("test/files/avif/sequence-without-pitm.avif").is_supported?)
+      assert_equal(false, MediaFile.open("test/files/avif/star-8bpc.avif").is_supported?)
+
+      # XXX These should be unsupported, but aren't.
+      # assert_equal(false, MediaFile.open("test/files/avif/alpha_video.avif").is_supported?)
+      # assert_equal(false, MediaFile.open("test/files/avif/plum-blossom-small-profile0.8bpc.yuv420.alpha-full.avif").is_supported?)
+      # assert_equal(false, MediaFile.open("test/files/avif/kimono.mirror-horizontal.avif").is_supported?)
+    end
+
+    should "detect animated files" do
+      assert_equal(true, MediaFile.open("test/files/avif/sequence-with-pitm.avif").is_animated?)
+      assert_equal(true, MediaFile.open("test/files/avif/sequence-without-pitm.avif").is_animated?)
+      assert_equal(true, MediaFile.open("test/files/avif/alpha_video.avif").is_animated?)
+      assert_equal(true, MediaFile.open("test/files/avif/star-8bpc.avif").is_animated?)
+
+      assert_equal(48, MediaFile.open("test/files/avif/sequence-with-pitm.avif").frame_count)
+      assert_equal(95, MediaFile.open("test/files/avif/sequence-without-pitm.avif").frame_count)
+      assert_equal(48, MediaFile.open("test/files/avif/alpha_video.avif").frame_count)
+      assert_equal(5, MediaFile.open("test/files/avif/star-8bpc.avif").frame_count)
+    end
+
+    should "detect static images with an auxiliary image sequence" do
+      assert_equal(true, MediaFile.open("test/files/avif/sequence-with-pitm-avif-major.avif").metadata.is_animated_avif?)
+      assert_equal(false, MediaFile.open("test/files/avif/sequence-with-pitm-avif-major.avif").is_animated?)
+      assert_equal(1, MediaFile.open("test/files/avif/sequence-with-pitm-avif-major.avif").frame_count)
+    end
+
+    should "detect rotated images" do
+      assert_equal(true, MediaFile.open("test/files/avif/kimono.rotate90.avif").metadata.is_rotated?)
+    end
+
+    should "detect monochrome images" do
+      assert_equal(true, MediaFile.open("test/files/avif/fox.profile0.8bpc.yuv420.monochrome.avif").metadata.is_greyscale?)
+    end
+
+    should "be able to generate a preview" do
+      assert_equal([180, 75], MediaFile.open("test/files/avif/hdr_cosmos01000_cicp9-16-9_yuv420_limited_qp40.avif").preview(180, 180).dimensions)
+      assert_equal([180, 75], MediaFile.open("test/files/avif/hdr_cosmos01000_cicp9-16-9_yuv444_full_qp40.avif").preview(180, 180).dimensions)
+      assert_equal([180, 135], MediaFile.open("test/files/avif/paris_icc_exif_xmp.avif").preview(180, 180).dimensions)
+      assert_equal([180, 180], MediaFile.open("test/files/avif/Image grid example.avif").preview(180, 180).dimensions)
+      assert_equal([180, 120], MediaFile.open("test/files/avif/fox.profile0.8bpc.yuv420.monochrome.avif").preview(180, 180).dimensions)
+      assert_equal([180, 123], MediaFile.open("test/files/avif/tiger_3layer_1res.avif").preview(180, 180).dimensions)
+    end
+  end
+
   context "a corrupt GIF" do
     should "still read the metadata" do
       @file = MediaFile.open("test/files/test-corrupt.gif")

test/unit/post_test.rb

@@ -1217,12 +1217,19 @@ class PostTest < ActiveSupport::TestCase
       end
 
       context "a greyscale image missing the greyscale tag" do
-        should "automatically add the greyscale tag" do
+        should "automatically add the greyscale tag for a monochrome JPEG file" do
           @media_asset = MediaAsset.upload!("test/files/test-grey-no-profile.jpg")
           @post.update!(md5: @media_asset.md5)
           @post.reload.update!(tag_string: "tagme")
           assert_equal("greyscale tagme", @post.tag_string)
         end
+
+        should "automatically add the greyscale tag for a monochrome AVIF file" do
+          @media_asset = MediaAsset.upload!("test/files/avif/fox.profile0.8bpc.yuv420.monochrome.avif")
+          @post.update!(md5: @media_asset.md5)
+          @post.reload.update!(tag_string: "tagme")
+          assert_equal("greyscale tagme", @post.tag_string)
+        end
       end
 
       context "an exif-rotated image missing the exif_rotation tag" do