jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

uploads: move thumbnail generation code to MediaFile.

Browse Source 
* Move image thumbnail generation code to MediaFile::Image.
* Move video thumbnail generation code to MediaFile::Video.
* Move ugoira->webm conversion code to MediaFile::Ugoira.

This separates thumbnail generation from the upload process so that it's
possible to generate thumbnails outside of uploads.
This commit is contained in:
evazion
2020-05-18 00:31:12 -05:00
parent 24c53172db
commit 45064853de
15 changed files with 269 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/factories/upload.rb
View File

@@ -17,7 +17,7 @@ FactoryBot.define do
factory(:ugoira_upload) do
file do
f = Tempfile.new
IO.copy_stream("#{Rails.root}/test/fixtures/ugoira.zip", f.path)
IO.copy_stream("#{Rails.root}/test/files/ugoira.zip", f.path)
ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "ugoira.zip")
end
end

7
test/files/ugoira.json Normal file
View File

@@ -0,0 +1,7 @@
[
{"file": "000000.jpg", "delay": 200},
{"file": "000001.jpg", "delay": 200},
{"file": "000002.jpg", "delay": 200},
{"file": "000003.jpg", "delay": 200},
{"file": "000004.jpg", "delay": 250}
]

0
test/fixtures/ugoira.zip → test/files/ugoira.zip
View File

43
test/unit/media_file_test.rb
View File

@@ -82,4 +82,47 @@ class MediaFileTest < ActiveSupport::TestCase
should "determine the correct filesize for a jpeg file" do
assert_equal(28086, MediaFile.open("test/files/test.jpg").file_size)
end
context "#preview" do
should "generate a preview image" do
assert_equal([150, 101], MediaFile.open("test/files/test.jpg").preview(150, 150).dimensions)
assert_equal([113, 150], MediaFile.open("test/files/test.png").preview(150, 150).dimensions)
assert_equal([150, 150], MediaFile.open("test/files/test.gif").preview(150, 150).dimensions)
assert_equal([150, 150], MediaFile.open("test/files/test-512x512.webm").preview(150, 150).dimensions)
assert_equal([150, 150], MediaFile.open("test/files/test-300x300.mp4").preview(150, 150).dimensions)
end
should "be able to fit to width only" do
assert_equal([400, 268], MediaFile.open("test/files/test.jpg").preview(400, nil).dimensions)
end
end
context "#crop" do
should "generate a cropped preview image" do
assert_equal([150, 150], MediaFile.open("test/files/test.jpg").crop(150, 150).dimensions)
assert_equal([150, 150], MediaFile.open("test/files/test.png").crop(150, 150).dimensions)
assert_equal([150, 150], MediaFile.open("test/files/test.gif").crop(150, 150).dimensions)
assert_equal([150, 150], MediaFile.open("test/files/test-512x512.webm").crop(150, 150).dimensions)
assert_equal([150, 150], MediaFile.open("test/files/test-300x300.mp4").crop(150, 150).dimensions)
end
end
context "for a ugoira" do
setup do
skip unless MediaFile::Ugoira.conversion_enabled?
frame_data = JSON.parse(File.read("test/files/ugoira.json"))
@ugoira = MediaFile.open("test/files/ugoira.zip", frame_data: frame_data)
end
should "generate a preview" do
assert_equal([60, 60], @ugoira.preview(150, 150).dimensions)
assert_equal([150, 150], @ugoira.crop(150, 150).dimensions)
end
should "convert to a webm" do
webm = @ugoira.convert
assert_equal(:webm, webm.file_ext)
assert_equal([60, 60], webm.dimensions)
end
end
end

24
test/unit/pixiv_ugoira_converter_test.rb
View File

@@ -1,24 +0,0 @@
require "test_helper"
class PixivUgoiraConverterTest < ActiveSupport::TestCase
context "An ugoira converter" do
setup do
@zipfile = upload_file("test/fixtures/ugoira.zip").tempfile
@frame_data = [
{"file" => "000000.jpg", "delay" => 200},
{"file" => "000001.jpg", "delay" => 200},
{"file" => "000002.jpg", "delay" => 200},
{"file" => "000003.jpg", "delay" => 200},
{"file" => "000004.jpg", "delay" => 250}
]
end
should "output to webm" do
skip "ffmpeg is not installed" unless PixivUgoiraConverter.enabled?
sample_file = PixivUgoiraConverter.generate_webm(@zipfile, @frame_data)
preview_file = PixivUgoiraConverter.generate_preview(@zipfile)
assert_operator(sample_file.size, :>, 1_000)
assert_operator(preview_file.size, :>, 0)
end
end
end

229
test/unit/upload_service_test.rb
View File

@@ -15,8 +15,6 @@ class UploadServiceTest < ActiveSupport::TestCase
}
context "::Utils" do
subject { UploadService::Utils }
context "#get_file_for_upload" do
context "for a non-source site" do
setup do
@@ -26,10 +24,8 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "work on a jpeg" do
file = subject.get_file_for_upload(@upload)
file = UploadService::Utils.get_file_for_upload(@upload)
assert_operator(File.size(file.path), :>, 0)
file.close
end
end
@@ -43,13 +39,9 @@ class UploadServiceTest < ActiveSupport::TestCase
should "work on an ugoira url" do
begin
file = subject.get_file_for_upload(@upload)
file = UploadService::Utils.get_file_for_upload(@upload)
assert_operator(File.size(file.path), :>, 0)
file.close
rescue Net::OpenTimeout
skip "network problems"
end
end
end
@@ -64,15 +56,12 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "work on an ugoira url" do
skip unless PixivUgoiraConverter.enabled?
file = subject.get_file_for_upload(@upload)
file = UploadService::Utils.get_file_for_upload(@upload)
assert_not_nil(@upload.context["ugoira"])
assert_operator(File.size(file.path), :>, 0)
file.close
rescue Net::OpenTimeout
skip "network failure"
end
end
end
@@ -84,14 +73,14 @@ class UploadServiceTest < ActiveSupport::TestCase
context "with an original_post_id" do
should "run" do
subject.expects(:distribute_files).times(3)
subject.process_file(@upload, @upload.file.tempfile, original_post_id: 12345)
UploadService::Utils.expects(:distribute_files).times(3)
UploadService::Utils.process_file(@upload, @upload.file.tempfile, original_post_id: 12345)
end
end
should "run" do
subject.expects(:distribute_files).times(3)
subject.process_file(@upload, @upload.file.tempfile)
UploadService::Utils.expects(:distribute_files).times(3)
UploadService::Utils.process_file(@upload, @upload.file.tempfile)
assert_equal("jpg", @upload.file_ext)
assert_equal(28086, @upload.file_size)
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
@@ -99,184 +88,6 @@ class UploadServiceTest < ActiveSupport::TestCase
assert_equal(500, @upload.image_width)
end
end
context ".generate_resizes" do
context "for an ugoira" do
setup do
context = UGOIRA_CONTEXT
@file = upload_file("test/fixtures/ugoira.zip")
@upload = mock
@upload.stubs(:is_video?).returns(false)
@upload.stubs(:is_ugoira?).returns(true)
@upload.stubs(:context).returns(context)
end
should "generate a preview and a video" do
skip unless PixivUgoiraConverter.enabled?
preview, crop, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(crop.path), :>, 0)
assert_operator(File.size(sample.path), :>, 0)
assert_equal(60, MediaFile.open(preview).width)
assert_equal(60, MediaFile.open(preview).height)
assert_equal(150, MediaFile.open(crop).width)
assert_equal(150, MediaFile.open(crop).height)
preview.close
preview.unlink
sample.close
sample.unlink
end
end
context "for a video" do
teardown do
@file.close
end
context "for an mp4" do
setup do
@file = upload_file("test/files/test-300x300.mp4")
@upload = mock
@upload.stubs(:is_video?).returns(true)
@upload.stubs(:is_ugoira?).returns(false)
end
should "generate a video" do
preview, crop, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(crop.path), :>, 0)
assert_equal(150, MediaFile.open(preview).width)
assert_equal(150, MediaFile.open(preview).height)
assert_equal(150, MediaFile.open(crop).width)
assert_equal(150, MediaFile.open(crop).height)
preview.close
preview.unlink
crop.close
crop.unlink
end
end
context "for a webm" do
setup do
@file = upload_file("test/files/test-512x512.webm")
@upload = mock
@upload.stubs(:is_video?).returns(true)
@upload.stubs(:is_ugoira?).returns(false)
end
should "generate a video" do
preview, crop, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(crop.path), :>, 0)
assert_equal(150, MediaFile.open(preview).width)
assert_equal(150, MediaFile.open(preview).height)
assert_equal(150, MediaFile.open(crop).width)
assert_equal(150, MediaFile.open(crop).height)
preview.close
preview.unlink
crop.close
crop.unlink
end
end
end
context "for an image" do
teardown do
@file.close
end
setup do
@upload = mock
@upload.stubs(:is_video?).returns(false)
@upload.stubs(:is_ugoira?).returns(false)
@upload.stubs(:is_image?).returns(true)
@upload.stubs(:image_width).returns(1200)
@upload.stubs(:image_height).returns(200)
end
context "for a jpeg" do
setup do
@file = upload_file("test/files/test.jpg")
end
should "generate a preview" do
preview, crop, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(crop.path), :>, 0)
assert_operator(File.size(sample.path), :>, 0)
preview.close
preview.unlink
sample.close
sample.unlink
end
end
context "for a png" do
setup do
@file = upload_file("test/files/test.png")
end
should "generate a preview" do
preview, crop, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(crop.path), :>, 0)
assert_operator(File.size(sample.path), :>, 0)
preview.close
preview.unlink
sample.close
sample.unlink
end
end
context "for a gif" do
setup do
@file = upload_file("test/files/test.png")
end
should "generate a preview" do
preview, crop, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(crop.path), :>, 0)
assert_operator(File.size(sample.path), :>, 0)
preview.close
preview.unlink
sample.close
sample.unlink
end
end
end
end
context ".generate_video_preview_for" do
context "for an mp4" do
setup do
@path = "test/files/test-300x300.mp4"
@video = FFMPEG::Movie.new(@path)
end
should "generate a video" do
sample = subject.generate_video_preview_for(@video, 100, 100)
assert_operator(File.size(sample.path), :>, 0)
sample.close
sample.unlink
end
end
context "for a webm" do
setup do
@path = "test/files/test-512x512.webm"
@video = FFMPEG::Movie.new(@path)
end
should "generate a video" do
sample = subject.generate_video_preview_for(@video, 100, 100)
assert_operator(File.size(sample.path), :>, 0)
sample.close
sample.unlink
end
end
end
end
context "::Preprocessor" do
@@ -302,7 +113,7 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "download the file" do
@service = subject.new(source: @source, referer_url: @ref)
@service = UploadService::Preprocessor.new(source: @source, referer_url: @ref)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(9800, @upload.file_size)
@@ -321,7 +132,7 @@ class UploadServiceTest < ActiveSupport::TestCase
should "download the file" do
begin
@service = subject.new(source: @source, referer_url: @ref)
@service = UploadService::Preprocessor.new(source: @source, referer_url: @ref)
@upload = @service.start!
rescue Net::OpenTimeout
skip "network failure"
@@ -341,9 +152,9 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "download the file" do
skip unless PixivUgoiraConverter.enabled?
skip unless MediaFile::Ugoira.conversion_enabled?
@service = subject.new(source: @source)
@service = UploadService::Preprocessor.new(source: @source)
begin
@upload = @service.start!
rescue Net::OpenTimeout
@@ -364,7 +175,7 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "download the file" do
@service = subject.new(source: @source)
@service = UploadService::Preprocessor.new(source: @source)
begin
@upload = @service.start!
rescue Net::OpenTimeout
@@ -386,7 +197,7 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "work for a video" do
@service = subject.new(source: @source)
@service = UploadService::Preprocessor.new(source: @source)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_not_nil(@upload.md5)
@@ -405,7 +216,7 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "leave the upload in an error state" do
@service = subject.new(source: @source)
@service = UploadService::Preprocessor.new(source: @source)
@upload = @service.start!
assert_match(/error:/, @upload.status)
end
@@ -413,7 +224,7 @@ class UploadServiceTest < ActiveSupport::TestCase
context "for an invalid content type" do
should "fail" do
upload = subject.new(source: "http://www.example.com").start!
upload = UploadService::Preprocessor.new(source: "http://www.example.com").start!
assert_match(/\Aerror:.*File ext is invalid/, upload.status)
end
end
@@ -429,7 +240,7 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "overwrite the attributes" do
@service = subject.new(source: @source, rating: 'e')
@service = UploadService::Preprocessor.new(source: @source, rating: 'e')
@upload = @service.start!
@service.finish!
@upload.reload
@@ -737,7 +548,7 @@ class UploadServiceTest < ActiveSupport::TestCase
context "a post that is replaced by a ugoira" do
should "save the frame data" do
skip "ffmpeg not installed" unless PixivUgoiraConverter.enabled?
skip unless MediaFile::Ugoira.conversion_enabled?
begin
as_user { @post.replace!(replacement_url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") }
@post.reload
@@ -751,8 +562,6 @@ class UploadServiceTest < ActiveSupport::TestCase
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source)
assert_equal([{"delay" => 125, "file" => "000000.jpg"}, {"delay" => 125, "file" => "000001.jpg"}], @post.pixiv_ugoira_frame_data.data)
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
@@ -760,7 +569,7 @@ class UploadServiceTest < ActiveSupport::TestCase
context "a post that is replaced to another file then replaced back to the original file" do
should "not delete the original files" do
begin
skip unless PixivUgoiraConverter.enabled?
skip unless MediaFile::Ugoira.conversion_enabled?
@post.unstub(:queue_delete_files)
# this is called thrice to delete the file for 62247364
@@ -802,7 +611,7 @@ class UploadServiceTest < ActiveSupport::TestCase
# swap the images between @post1 and @post2.
begin
as_user do
skip unless PixivUgoiraConverter.enabled?
skip unless MediaFile::Ugoira.conversion_enabled?
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")