From 0e901b2f842dd2208cd2b504bf359a675495f02e Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 27 Sep 2021 04:41:40 -0500 Subject: [PATCH] media file: get duration of animated GIFs, PNGs, and ugoiras. Add methods to MediaFile to calculate the duration, frame count, and frame rate of animated GIFs, PNGs, Ugoiras, and videos. Some considerations: * It's possible to have a GIF or PNG that's technically animated but just has one frame. These are treated as non-animated images. * It's possible to have an animated GIF that has an unspecified frame rate. In this case we assume the frame rate is 10 FPS; this is browser dependent and may not be correct. * Animated GIFs, PNGs, and Ugoiras all support variable frame rates. Technically, each frame has a separate delay, and the delays can be different frame-to-frame. We report only the average frame rate. * Getting the duration of an APNG is surprisingly hard. Most tools don't have good support for APNGs since it's a rare and non-standardized format. The best we can do is get the frame count using ExifTool and the frame rate using ffprobe, then calculate the duration from that. --- app/logical/ffmpeg.rb | 24 +++++++++++--- app/logical/media_file.rb | 29 +++++++++++++---- app/logical/media_file/image.rb | 55 +++++++++++++++++++++++++++++--- app/logical/media_file/ugoira.rb | 16 ++++++++++ app/logical/media_file/video.rb | 2 +- test/unit/media_file_test.rb | 48 +++++++++++++++++++++++++++- 6 files changed, 156 insertions(+), 18 deletions(-) diff --git a/app/logical/ffmpeg.rb b/app/logical/ffmpeg.rb index 8dc0ea6ee..07e43147f 100644 --- a/app/logical/ffmpeg.rb +++ b/app/logical/ffmpeg.rb @@ -40,27 +40,41 @@ class FFmpeg end def width - video_channels.first[:width] + video_streams.first[:width] end def height - video_channels.first[:height] + video_streams.first[:height] end def duration metadata.dig(:format, :duration).to_f end - def video_channels + def frame_count + if video_streams.first.has_key?(:nb_frames) + video_streams.first[:nb_frames].to_i + else + (duration * frame_rate).to_i + end + end + + def frame_rate + rate = video_streams.first[:avg_frame_rate] # "100/57" + numerator, denominator = rate.split("/") + (numerator.to_f / denominator.to_f) + end + + def video_streams metadata[:streams].select { |stream| stream[:codec_type] == "video" } end - def audio_channels + def audio_streams metadata[:streams].select { |stream| stream[:codec_type] == "audio" } end def has_audio? - audio_channels.present? + audio_streams.present? end def shell!(command) diff --git a/app/logical/media_file.rb b/app/logical/media_file.rb index d22ee19ae..d7991c61b 100644 --- a/app/logical/media_file.rb +++ b/app/logical/media_file.rb @@ -135,7 +135,26 @@ class MediaFile # @return [Boolean] true if the file is animated. Note that GIFs and PNGs may be animated. def is_animated? - is_video? + is_video? || frame_count.to_i > 1 + end + + # @return [Float, nil] the duration of the video or animation in seconds, or + # nil if not a video or animation, or the duration is unknown. + def duration + nil + end + + # @return [Float, nil] the number of frames in the video or animation, or nil + # if not a video or animation. + def frame_count + nil + end + + # @return [Float, nil] the average frame rate of the video or animation, or + # nil if not a video or animation. Note that GIFs and PNGs can have a + # variable frame rate. + def frame_rate + nil end # @return [Boolean] true if the file has an audio track. The track may not be audible. @@ -143,11 +162,6 @@ class MediaFile false end - # @return [Float] the duration of the video or animation, in seconds. - def duration - 0.0 - end - # Return a preview of the file, sized to fit within the given width and # height (preserving the aspect ratio). # @@ -179,6 +193,9 @@ class MediaFile file_size: file_size, md5: md5, is_corrupt?: is_corrupt?, + duration: duration, + frame_count: frame_count, + frame_rate: frame_rate, metadata: metadata }.stringify_keys end diff --git a/app/logical/media_file/image.rb b/app/logical/media_file/image.rb index 297c84cd4..1734e16bc 100644 --- a/app/logical/media_file/image.rb +++ b/app/logical/media_file/image.rb @@ -23,8 +23,49 @@ class MediaFile::Image < MediaFile true end - def is_animated? - is_animated_gif? || is_animated_png? + def duration + if is_animated_gif? + if metadata.has_key?("GIF:Duration") + # GIF:Duration => "9.03 s" + metadata["GIF:Duration"].to_f + else + # If GIF:Duration is absent then it means the GIF has an unlimited + # framerate. In this situation we assume the browser will play the GIF + # at 10 FPS; this is browser dependent. + # + # A GIF consists of a series of frames, each frame having a separate frame + # delay. The duration of the GIF is the sum of each frame delay. If the frame + # delay of each frame is zero, then it means the GIF has an unlimited framerate + # and should be played as fast as possible. In reality, browsers cap the + # framerate to around 10FPS. + (frame_count * (1.0/10.0)).round(2) + end + elsif is_animated_png? + frame_count.to_f / frame_rate + else + nil + end + end + + def frame_count + if file_ext == :gif + image.get("n-pages") + elsif file_ext == :png + metadata.fetch("PNG:AnimationFrames", 1) + else + nil + end + end + + def frame_rate + if is_animated_gif? + frame_count / duration + elsif is_animated_png? + # XXX we have to resort to ffprobe to get the frame rate because libvips and exiftool can't get it. + video.frame_rate + else + nil + end end def channels @@ -62,11 +103,11 @@ class MediaFile::Image < MediaFile end def is_animated_gif? - file_ext == :gif && image.get("n-pages") > 1 + file_ext == :gif && is_animated? end def is_animated_png? - file_ext == :png && metadata.fetch("PNG:AnimationFrames", 1) > 1 + file_ext == :png && is_animated? end # @return [Vips::Image] the Vips image object for the file @@ -74,5 +115,9 @@ class MediaFile::Image < MediaFile Vips::Image.new_from_file(file.path, fail: true).autorot end - memoize :image, :dimensions, :is_corrupt?, :is_animated_gif?, :is_animated_png? + def video + FFmpeg.new(file) + end + + memoize :image, :video, :dimensions, :is_corrupt?, :is_animated_gif?, :is_animated_png? end diff --git a/app/logical/media_file/ugoira.rb b/app/logical/media_file/ugoira.rb index aefc19bf5..893457731 100644 --- a/app/logical/media_file/ugoira.rb +++ b/app/logical/media_file/ugoira.rb @@ -32,6 +32,22 @@ class MediaFile::Ugoira < MediaFile preview_frame.crop(width, height) end + def duration + (frame_delays.sum / 1000.0) + end + + def frame_count + frame_data.count + end + + def frame_rate + frame_count / duration + end + + def frame_delays + frame_data.map { |frame| frame["delay"] } + end + # Convert a ugoira to a webm. # XXX should take width and height and resize image def convert diff --git a/app/logical/media_file/video.rb b/app/logical/media_file/video.rb index b981c432e..3903c8cf7 100644 --- a/app/logical/media_file/video.rb +++ b/app/logical/media_file/video.rb @@ -3,7 +3,7 @@ # # @see https://github.com/streamio/streamio-ffmpeg class MediaFile::Video < MediaFile - delegate :duration, :has_audio?, to: :video + delegate :duration, :frame_count, :frame_rate, :has_audio?, to: :video def dimensions [video.width, video.height] diff --git a/test/unit/media_file_test.rb b/test/unit/media_file_test.rb index 8b1cd8e90..9943124bd 100644 --- a/test/unit/media_file_test.rb +++ b/test/unit/media_file_test.rb @@ -166,6 +166,12 @@ class MediaFileTest < ActiveSupport::TestCase assert_equal([150, 150], @ugoira.crop(150, 150).dimensions) end + should "get the duration" do + assert_equal(1.05, @ugoira.duration) + assert_equal(4.76, @ugoira.frame_rate.round(2)) + assert_equal(5, @ugoira.frame_count) + end + should "convert to a webm" do webm = @ugoira.convert assert_equal(:webm, webm.file_ext) @@ -173,11 +179,32 @@ class MediaFileTest < ActiveSupport::TestCase end end - context "for a video" do + context "for an mp4 file " do should "detect videos with audio" do assert_equal(true, MediaFile.open("test/files/test-audio.mp4").has_audio?) assert_equal(false, MediaFile.open("test/files/test-300x300.mp4").has_audio?) end + + should "determine the duration of the video" do + file = MediaFile.open("test/files/test-audio.mp4") + assert_equal(1.002667, file.duration) + assert_equal(10, file.frame_rate) + assert_equal(10, file.frame_count) + + file = MediaFile.open("test/files/test-300x300.mp4") + assert_equal(5.7, file.duration) + assert_equal(1.75, file.frame_rate.round(2)) + assert_equal(10, file.frame_count) + end + end + + context "for a webm file" do + should "determine the duration of the video" do + file = MediaFile.open("test/files/test-512x512.webm") + assert_equal(0.48, file.duration) + assert_equal(50, file.frame_rate) + assert_equal(24, file.frame_count) + end end context "a compressed SWF file" do @@ -190,6 +217,15 @@ class MediaFileTest < ActiveSupport::TestCase end end + context "an animated GIF file" do + should "determine the duration of the animation" do + file = MediaFile.open("test/files/test-animated-86x52.gif") + assert_equal(0.4, file.duration) + assert_equal(10, file.frame_rate) + assert_equal(4, file.frame_count) + end + end + context "a PNG file" do context "that is not animated" do should "not be detected as animated" do @@ -197,6 +233,9 @@ class MediaFileTest < ActiveSupport::TestCase assert_equal(false, file.is_corrupt?) assert_equal(false, file.is_animated?) + assert_nil(file.duration) + assert_nil(file.frame_rate) + assert_equal(1, file.frame_count) end end @@ -206,6 +245,9 @@ class MediaFileTest < ActiveSupport::TestCase assert_equal(false, file.is_corrupt?) assert_equal(true, file.is_animated?) + assert_equal(9, file.duration) + assert_equal(0.33, file.frame_rate.round(2)) + assert_equal(3, file.frame_count) end end @@ -215,6 +257,9 @@ class MediaFileTest < ActiveSupport::TestCase assert_equal(false, file.is_corrupt?) assert_equal(false, file.is_animated?) + assert_nil(file.duration) + assert_nil(file.frame_rate) + assert_equal(1, file.frame_count) end end @@ -239,6 +284,7 @@ class MediaFileTest < ActiveSupport::TestCase file = MediaFile.open("test/files/apng/actl_zero_frames.png") assert_equal(false, file.is_corrupt?) assert_equal(false, file.is_animated?) + assert_equal(0, file.frame_count) end end end