From ef28576673d3080df2807842f8415cabe09db095 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 5 Sep 2021 05:43:59 -0500 Subject: [PATCH] Fix #3400: Smarter thumbnail generation for videos --- app/logical/ffmpeg.rb | 25 +++++++++++++++++++++++++ app/logical/media_file/image.rb | 12 ++++++++---- app/logical/media_file/ugoira.rb | 7 +++---- app/logical/media_file/video.rb | 4 +--- config/initializers/inflections.rb | 1 + test/files/valid_ugoira.zip | Bin 6663 -> 0 bytes test/unit/media_file_test.rb | 16 ++++++++++++---- test/unit/upload_service_test.rb | 2 -- 8 files changed, 50 insertions(+), 17 deletions(-) create mode 100644 app/logical/ffmpeg.rb delete mode 100644 test/files/valid_ugoira.zip diff --git a/app/logical/ffmpeg.rb b/app/logical/ffmpeg.rb new file mode 100644 index 000000000..24f30045c --- /dev/null +++ b/app/logical/ffmpeg.rb @@ -0,0 +1,25 @@ +class FFmpeg + attr_reader :file + + # Operate on a file with FFmpeg. + # @param file [File, String] a webm, mp4, gif, or apng file + def initialize(file) + @file = file.is_a?(String) ? File.open(file) : file + end + + # Generate a .jpg preview image for a video or animation. Generates + # thumbnails intelligently by avoiding blank frames. + # + # @return [MediaFile] the preview image + def smart_video_preview + vp = Tempfile.new(["video-preview", ".jpg"], binmode: true) + + # https://ffmpeg.org/ffmpeg.html#Main-options + # https://ffmpeg.org/ffmpeg-filters.html#thumbnail + ffmpeg_out, status = Open3.capture2e("ffmpeg -i #{file.path} -vf thumbnail=300 -frames:v 1 -y #{vp.path}") + raise "ffmpeg failed: #{ffmpeg_out}" if !status.success? + Rails.logger.debug(ffmpeg_out) + + MediaFile.open(vp) + end +end diff --git a/app/logical/media_file/image.rb b/app/logical/media_file/image.rb index e65508602..dab6a5dab 100644 --- a/app/logical/media_file/image.rb +++ b/app/logical/media_file/image.rb @@ -38,11 +38,15 @@ class MediaFile::Image < MediaFile # @see https://github.com/jcupitt/libvips/wiki/HOWTO----Image-shrinking # @see http://jcupitt.github.io/libvips/API/current/Using-vipsthumbnail.md.html def preview(width, height) - output_file = Tempfile.new(["image-preview", ".jpg"]) - resized_image = image.thumbnail_image(width, height: height, **THUMBNAIL_OPTIONS) - resized_image.jpegsave(output_file.path, **JPEG_OPTIONS) + if is_animated? + FFmpeg.new(file).smart_video_preview + else + output_file = Tempfile.new(["image-preview", ".jpg"]) + resized_image = image.thumbnail_image(width, height: height, **THUMBNAIL_OPTIONS) + resized_image.jpegsave(output_file.path, **JPEG_OPTIONS) - MediaFile::Image.new(output_file) + MediaFile::Image.new(output_file) + end end def crop(width, height) diff --git a/app/logical/media_file/ugoira.rb b/app/logical/media_file/ugoira.rb index 19270f754..aefc19bf5 100644 --- a/app/logical/media_file/ugoira.rb +++ b/app/logical/media_file/ugoira.rb @@ -36,6 +36,7 @@ class MediaFile::Ugoira < MediaFile # XXX should take width and height and resize image def convert raise NotImplementedError, "can't convert ugoira to webm: ffmpeg or mkvmerge not installed" unless self.class.videos_enabled? + raise RuntimeError, "can't convert ugoira to webm: no ugoira frame data was provided" unless frame_data.present? Dir.mktmpdir("ugoira-#{md5}") do |tmpdir| output_file = Tempfile.new(["ugoira-conversion", ".webm"], binmode: true) @@ -87,10 +88,8 @@ class MediaFile::Ugoira < MediaFile end def preview_frame - tempfile = Tempfile.new("ugoira-preview", binmode: true) - zipfile.entries.first.extract(tempfile.path) { true } # 'true' means overwrite the existing tempfile. - MediaFile.open(tempfile) + FFmpeg.new(convert).smart_video_preview end - memoize :zipfile, :preview_frame, :dimensions + memoize :zipfile, :preview_frame, :dimensions, :convert end diff --git a/app/logical/media_file/video.rb b/app/logical/media_file/video.rb index d540483d5..13afbd7fb 100644 --- a/app/logical/media_file/video.rb +++ b/app/logical/media_file/video.rb @@ -32,9 +32,7 @@ class MediaFile::Video < MediaFile end def preview_frame - vp = Tempfile.new(["video-preview", ".jpg"], binmode: true) - video.screenshot(vp.path, seek_time: 0) - MediaFile.open(vp.path) + FFmpeg.new(file).smart_video_preview end memoize :video, :preview_frame, :dimensions, :duration, :has_audio? diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index 09961dda5..bb36bfc5f 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -5,6 +5,7 @@ # locales as you wish. All of these examples are active by default: ActiveSupport::Inflector.inflections(:en) do |inflect| inflect.uncountable "general" + inflect.acronym "FFmpeg" # inflect.plural /^(ox)$/i, '\1en' # inflect.singular /^(ox)en/i, '\1' # inflect.irregular 'person', 'people' diff --git a/test/files/valid_ugoira.zip b/test/files/valid_ugoira.zip deleted file mode 100644 index 463bed1c80444b39af01e33d6bf82878495ae99d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6663 zcmWIWW@Zs#0D;H5U0qYWJu1`L7#Kj9i-Cc`01Wi93ex}I`2T=Gz{}0kje(Jok->w( zgMs1yZ3brsW+o;OVrFJ$VP>6&2;-7ncwdkq{CQ6#*H-$jrjR%EHRY%E~Fi%grl7GWdUhL6C#NhQWrJQILU2 zkdaxC@&6G9X$D3HCT2!PknfowQ9xswBzrzk82)U5&X~oJ?}c#ozlAkww>3glDYGA#ieIk z6Fpv=`fU#kmwfg_BVy7@Ghac8O?|7BzW!%Wn4KiP&T(hSjBUB@J-6&XEWP72*~n!1 z8R`CCL5Z$zQ#vNruCq>-krr`%vufoj{SbwzFThSyv|iYn68SeM=xUmg!Ku!kJxAx9 z>R9=;R7k`jqNwAF(j@&9WwlsUr$1d67g;pQ-_Y$|eM8OItEOt#zWHJscBPfN)>cjW zCBd~r&{Ou!oYhO-g!goYT-m))P*bo#WZl6Z;hEd^xx3!g{`Gc$>d|8{jUNndC9Y`? zH#ud0OQk+&Vu7+LS91SvSxJvuCKt>PvL}E8?C|Vdwp~6PpPspQD^;JClJ>V!2#R$( zdo_1U=#!JunTKD^Ss!+?J~8w{m}U4B3qIqVkdM#KpUex3-rDPOTw8wL%^PNXk_Rju zJ$8uL~i`? z`L%yrdbw7V+p5D!UYinT~JB_??T8=$sLudW)B~IJU5wbGs~JI?-w7RWTx^Y z)$QF}IbWHCqc?rr`5&pAnN+;_kfMNI)+?`9T^UAgq3&6qDNx1@8%+{2edW32A|XAn{rtp0fW!iDxc`@9PO z{qOdFkNok+&Rn>@w`?+|IY8oH@}>}=1HCAG7I1R zK04m6?GmRaOH1UL9^{!S#2>TJ)U2g4VOF?oIFs;Gw6u$4b z%UYhcCn}M-TV7`xDOy}oJYHB4Xl^B(R(nA){O!9q$L_GhcY0T^KXIr3b)o9pzJ$kO zI=&8pnmjrS8v>&g)f+>1raceeo7Km-!$Y#v{C9I;gY%KH>uFDt)B_^~wC3L7KG7U4 zxono(OwkAXNs|m!9AJr)j^&NsUCG^~=)MO~`)W z$fF(6%$u7G;$1|a#ccEMb*TK%9YaJ z;0xd=024*gLIqjNCT+5*oaV!QSIlPf4o1zu)fd+*M|@2Vy`!{K;;YNkz;%yX`K5lg zPspzme{?Qj`cJj{x1#P}I<>}>*H7QV?%V~R%om(38^e|_Q%IY+ULkn@LaCm2-!?j5 zn-QY3j5W}Gdd%f}k(yHvPxK6my09fS`fy0fWW~_C6JOunrL<^Ksh? zjzuQh%%y#9mW;3YH{=?|7|EVmQv6r(ACq(xznV?a%1{2hz5O>-ZPXX;P1!!%S?*Lx zhJVyP*z5eY-p|radtzTgYS=%O+1ek@=Y2c7!;ha|r?da)@|o6(O_tjI z77vmlPq#HCPnh(#O$hd4BSRj;Be*clW#U?LW9@_s_RG2yAexNszC zYu}_0=~IiPv_5@TIO6`*FMP$F{fk3OB>O_2@-g?^wwh;i^QZ2q)4BRRqQALkqn+-TeK!v6d!<7dm|MK`C;4clmJf8xEJ_1-nU%am+R*{M%( zIik7o$IP$wIp^MmZ0QYcJ2EweogrUyqC}{4=o1;W>X@t99I2d>EIQsTx^8~4Zpp*1 zw^NqO=|(Qw9UW-AKWgU3&#w#LF1MHZp`>+(h>9jzG~ZwuLQJj^&Cuyw_?&SsZa*OUVV zr8*BO6&_o?S9{WC?_-auef&H7gIvz|J^a!d&bNL4kq(Z%Ki<9&j=bG}d7kj=U+e$9 zwwG@|_9*}3#<1C+za4S=`rh-_xBYLe6CAgfVp_8^~0HmfV!I>WhiZyX0zp8s?+rA#?? zot|*e$T_gX+^uBUD$c&34wF5v^A@Govn<_lLR0;~yBEb8>r$M*3f>KsbJ+N=(>Am> zBxT~1`kkU(*Y~dDTL0i)?4NIU_+J#he;dUc?rQ!?oWa;?vBu;d_x;yXtca2{A<(@u~*qUQ|;?}M|Jrnzr$35#l}6iEYD5pSJyFzReBHN zTyk>~IXo1$hVgW6;(7ML^u-e?+qLy-*}E1u)yv;p?RmT3dw%*)*1Et+0f%PB`$+9_ zyMCbQq?V}RjYHEM|9MZa+o~ih`MPK;+tuPbA9g*vUD@?6G+?#So2>a-YXv8HJiL2A z+<9WMQz37FY{|txCg!E4exg~}Q?sqt_@?747!#(GJbqF3d zY~q+T$8@oD)G<@RwOL9^8K$w!4$uN$9=*~n$TqT+WMg0eVR)r$G*l~HDF$%I4Y>~p zu5>|NKrL{c%gVsW!oG8%;-4$~;-ZWKdq9_!i}_r)nc{wqi};y#@m|-w?bEHWcd8xRn>2|t;k&-9 zTHWbWc;i7p_@Wc*ORv6Jv1IQY{ge7K>Ku(%TNA{0tX;k7&3A2$Gs&~HE*#w58**EB z(u7qO-9p0bHw8i}f&(8+pBVZuKo4Ag8|vi0QvccVey#YW?fdgf;XCVfU&Hg?{;jj!9X?$(iE&$*cama%Gl^ z`0UaJZ$fVIfm#e~jSC(X&D#62Q}={nUyzoNoln<%&A_0=@@z>{G9hw%(@g&zNF<0^Egwtzw z=y(_?<$tZcC;Z8NXHi$zid`H^Wv>g2d@tWR&ZaW!PkM3IKHD^BxpULJFTB>B*zx{U zN@Vy<^;xezRqnX`ghh=_&%*wVc+{~^tKOI#JLGETJk8|v5_QS;tOX~_Cxx}T^aXCR zDG}0(jsK=1|H4EtB6QoM^IJ>z>=r%x;&tFuwI@L-n{wu6(u{GPRJF~I()1vLGj&F))iZS$@4436K-NGVa<_#%Nf+^NvI%2INn$j7fwLJovV2eB41#1uX* zF!o<|B;D+Q@$!8i<&_uhKB4|6XWPW*N;^wbF8iLU-5y;rHz#t6(Ia=2;tg)?qKtR@Z8YCu}q1~<*h1>8Ny^PJp;O;!fRp9{jt%U z6MDsD+*+4KWU#VGGl;kd3c9$6_z4OMnn))kc-d^7M_3_Rkv0c1dRvg=yty86PrtyWC+`RJahwXYDiYhbO-N={&!h9g{Qba zm?{FvpWu-qDYjJLp&{lUbB0h@y-Ye)R?Q7m+ZF-_!-^ zr7>%^eX8@iWiIuc`-n+#>Bhjrl9t?CdcJBp)Xr=@-fk|(`BS`3w&^C4Mn*dD{Uy*qd2Zv6Bz?i3afkhv`Ua8KXfsmm(X ztG^d52u4d{a(=}|*r!F$}zIMRKInJ6l#-k$X;iE#OJP+?lPjpLjO!wBahPD3r=A5>jKkui{ zlF~y*{!E@+p7qgHxHLU6Z)5)x-ShgVcjQhG5D*Y3nt$kdkLG^{*EN6nZ~xrax<5Pm z$NJCjf7WcM{`L26?c81Ec6-Y8oYpfw_Ho;~I%Vst>1LcOQf%kF+$1gM_VnO89@bae zO`hA>7#sJmIli*U`JnYK9$)n%7P`#ZX3moJ8C#Pg;+S~u-JE*r(4i+P?=D7f*5EAC z{k~zNanYGS!kb$+F(Cnd^!&l-^svQ;k%0k(;rYX4sOAq*Q2qcXT{r>C8zKUjN&5%` znb}}f0^Drk81|qrXJis#76GfrHnjnkU|?X_(g, 0) - - file.close end end end