jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

uploads: use storage manager to distribute files.

Browse Source 
Refactors the upload process to pass around temp files, rather than
passing around file paths and directly writing output to the local
filesystem. This way we can pass the storage manager the preview /
sample / original temp files, so it can deal with storage itself.

* Change Download::File#download! to return a temp file.

* Change DanbooruImageResizer and PixivUgoiraConverter to accept/return
  temp files instead of file paths.

* Change Upload#generate_resizes to return temp files for previews and samples.

* Change Upload#generate_resizes to generate ugoira .webm samples
  synchronously instead of asynchronously.
This commit is contained in:
evazion
2018-03-18 12:05:25 -05:00
parent 60dcfbfbdd
commit c76463f34d
8 changed files with 89 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/uploads_controller.rb
View File

@@ -6,7 +6,7 @@ class UploadsController < ApplicationController
@upload = Upload.new
@upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
if params[:url]
download = Downloads::File.new(params[:url], ".")
download = Downloads::File.new(params[:url])
@normalized_url, _, _ = download.before_download(params[:url], {})
@post = find_post_by_url(@normalized_url)

9
app/logical/danbooru_image_resizer.rb
View File

@@ -1,6 +1,6 @@
module DanbooruImageResizer
def resize(read_path, write_path, width, height, resize_quality = 90)
image = Magick::Image.read(read_path).first
def resize(file, width, height, resize_quality = 90)
image = Magick::Image.read(file.path).first
geometry = "#{width}x>"
if width == Danbooru.config.small_image_width
@@ -17,14 +17,15 @@ module DanbooruImageResizer
image = flatten(image, width, height)
image.strip!
image.write(write_path) do
output_file = Tempfile.new(binmode: true)
image.write("jpeg:" + output_file.path) do
self.quality = resize_quality
# setting PlaneInterlace enables progressive encoding for JPEGs
self.interlace = Magick::PlaneInterlace
end
image.destroy!
FileUtils.chmod(0664, write_path)
output_file
end
def flatten(image, width, height)

17
app/logical/downloads/file.rb
View File

@@ -3,9 +3,9 @@ module Downloads
class Error < Exception ; end
attr_reader :data, :options
attr_accessor :source, :original_source, :downloaded_source, :file_path
attr_accessor :source, :original_source, :downloaded_source
def initialize(source, file_path, options = {})
def initialize(source, options = {})
# source can potentially get rewritten in the course
# of downloading a file, so check it again
@source = source
@@ -14,9 +14,6 @@ module Downloads
# the URL actually downloaded after rewriting the original source.
@downloaded_source = nil
# where to save the download
@file_path = file_path
# we sometimes need to capture data from the source page
@data = {}
@@ -35,12 +32,13 @@ module Downloads
def download!
url, headers, @data = before_download(@source, @data)
::File.open(@file_path, "wb") do |out|
http_get_streaming(uncached_url(url, headers), out, headers)
end
output_file = Tempfile.new(binmode: true)
http_get_streaming(uncached_url(url, headers), output_file, headers)
@downloaded_source = url
@source = after_download(url)
output_file
end
def before_download(url, datums)
@@ -91,7 +89,8 @@ module Downloads
end
if res.success?
return
file.rewind
return file
else
raise Error.new("HTTP error code: #{res.code} #{res.message}")
end

29
app/logical/pixiv_ugoira_converter.rb
View File

@@ -1,13 +1,9 @@
class PixivUgoiraConverter
def self.convert(source_path, output_path, preview_path, frame_data)
folder = Zip::File.new(source_path)
write_webm(folder, output_path, frame_data)
write_preview(folder, preview_path)
RemoteFileManager.new(output_path).distribute
RemoteFileManager.new(preview_path).distribute
end
def self.generate_webm(ugoira_file, frame_data)
folder = Zip::File.new(ugoira_file.path)
output_file = Tempfile.new(binmode: true)
write_path = output_file.path
def self.write_webm(folder, write_path, frame_data)
Dir.mktmpdir do |tmpdir|
FileUtils.mkdir_p("#{tmpdir}/images")
folder.each_with_index do |file, i|
@@ -64,14 +60,17 @@ class PixivUgoiraConverter
return
end
end
output_file
end
def self.write_preview(folder, path)
Dir.mktmpdir do |tmpdir|
file = folder.first
temp_path = File.join(tmpdir, file.name)
file.extract(temp_path)
DanbooruImageResizer.resize(temp_path, path, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
end
def self.generate_preview(ugoira_file)
file = Tempfile.new(binmode: true)
zipfile = Zip::File.new(ugoira_file.path)
zipfile.entries.first.extract(file.path) { true } # 'true' means overwrite the existing tempfile.
DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
ensure
file.close!
end
end

22
app/logical/pixiv_ugoira_service.rb
View File

@@ -1,32 +1,10 @@
class PixivUgoiraService
attr_reader :width, :height, :frame_data, :content_type
def self.regen(post)
service = new()
service.load(
:is_ugoira => true,
:ugoira_frame_data => post.pixiv_ugoira_frame_data.data
)
service.generate_resizes(post.file_path, post.large_file_path, post.preview_file_path, false)
end
def save_frame_data(post)
PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id)
end
def generate_resizes(source_path, output_path, preview_path, delay = true)
# Run this a bit in the future to give the upload process time to move the file
if delay
PixivUgoiraConverter.delay(:queue => Socket.gethostname, :run_at => 10.seconds.from_now, :priority => -1).convert(source_path, output_path, preview_path, @frame_data)
else
PixivUgoiraConverter.convert(source_path, output_path, preview_path, @frame_data)
end
# since the resizes will be delayed, just touch the output file so the
# file distribution wont break
FileUtils.touch([output_path, preview_path])
end
def calculate_dimensions(source_path)
folder = Zip::File.new(source_path)
tempfile = Tempfile.new("ugoira-dimensions")

15
app/models/post.rb
View File

@@ -130,17 +130,10 @@ class Post < ApplicationRecord
Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true)
end
def distribute_files
if Danbooru.config.build_file_url(self) =~ /^http/
# this post is archived
RemoteFileManager.new(file_path).distribute_to_archive(Danbooru.config.build_file_url(self))
RemoteFileManager.new(preview_file_path).distribute if has_preview?
RemoteFileManager.new(large_file_path).distribute_to_archive(Danbooru.config.build_large_file_url(self)) if has_large?
else
RemoteFileManager.new(file_path).distribute
RemoteFileManager.new(preview_file_path).distribute if has_preview?
RemoteFileManager.new(large_file_path).distribute if has_large?
end
def distribute_files(file, sample_file, preview_file)
storage_manager.store_file(file, self, :original)
storage_manager.store_file(sample_file, self, :large) if sample_file.present?
storage_manager.store_file(preview_file, self, :preview) if preview_file.present?
end
def file_path_prefix

4
app/models/post_replacement.rb
View File

@@ -24,6 +24,8 @@ class PostReplacement < ApplicationRecord
end
def process!
upload = nil
transaction do
upload = Upload.create!(
file: replacement_file,
@@ -79,7 +81,7 @@ class PostReplacement < ApplicationRecord
# point of no return: these things can't be rolled back, so we do them
# only after the transaction successfully commits.
post.distribute_files
upload.distribute_files(post)
post.update_iqdb_async
end

174
app/models/upload.rb
View File

@@ -10,13 +10,11 @@ class Upload < ApplicationRecord
belongs_to :uploader, :class_name => "User"
belongs_to :post
before_validation :initialize_uploader, :on => :create
before_create :convert_cgi_file
after_destroy :delete_temp_file
validate :uploader_is_not_limited, :on => :create
validate :file_or_source_is_present, :on => :create
validate :rating_given
attr_accessible :file, :image_width, :image_height, :file_ext, :md5,
:file_size, :as_pending, :source, :file_path, :content_type, :rating,
:file_size, :as_pending, :source, :rating,
:tag_string, :status, :backtrace, :post_id, :md5_confirmation,
:parent_id, :server, :artist_commentary_title,
:artist_commentary_desc, :include_artist_commentary,
@@ -101,31 +99,36 @@ class Upload < ApplicationRecord
module ConversionMethods
def process_upload
CurrentUser.scoped(uploader, uploader_ip_addr) do
begin
update_attribute(:status, "processing")
self.source = source.to_s.strip
if is_downloadable?
self.downloaded_source, self.source = download_from_source(temp_file_path)
self.downloaded_source, self.source, self.file = download_from_source(source, referer_url)
else
self.file = self.file.tempfile
end
self.file_ext = file_header_to_file_ext(file_path)
self.file_ext = file_header_to_file_ext(file)
self.file_size = file.size
self.md5 = Digest::MD5.file(file.path).hexdigest
validate_file_content_type
calculate_hash(file_path)
validate_md5_uniqueness
validate_md5_confirmation
validate_video_duration
calculate_file_size(file_path)
self.tag_string = "#{tag_string} #{automatic_tags}"
self.image_width, self.image_height = calculate_dimensions
generate_resizes(file_path)
move_file
save
end
end
def create_post_from_upload
post = convert_to_post
post.distribute_files
distribute_files(post)
if post.save
create_artist_commentary(post) if include_artist_commentary?
ugoira_service.save_frame_data(post) if is_ugoira?
@@ -138,6 +141,14 @@ class Upload < ApplicationRecord
post
end
def distribute_files(post)
preview_file, sample_file = generate_resizes
post.distribute_files(file, sample_file, preview_file)
ensure
preview_file.try(:close!)
sample_file.try(:close!)
end
def process!(force = false)
@tries ||= 0
return if !force && status =~ /processing|completed|error/
@@ -157,9 +168,9 @@ class Upload < ApplicationRecord
rescue Exception => x
update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
nil
ensure
delete_temp_file
file.try(:close!)
end
def ugoira_service
@@ -194,23 +205,6 @@ class Upload < ApplicationRecord
end
module FileMethods
def delete_temp_file(path = nil)
FileUtils.rm_f(path || temp_file_path)
end
def move_file
FileUtils.mv(file_path, md5_file_path)
end
def calculate_file_size(source_path)
self.file_size = File.size(source_path)
end
# Calculates the MD5 based on whatever is in temp_file_path
def calculate_hash(source_path)
self.md5 = Digest::MD5.file(source_path).hexdigest
end
def is_image?
%w(jpg gif png).include?(file_ext)
end
@@ -232,52 +226,43 @@ class Upload < ApplicationRecord
end
def is_animated_gif?
file_ext == "gif" && Magick::Image.ping(file_path).length > 1
file_ext == "gif" && Magick::Image.ping(file.path).length > 1
end
def is_animated_png?
file_ext == "png" && APNGInspector.new(file_path).inspect!.animated?
file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
end
end
module ResizerMethods
def generate_resizes(source_path)
generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85)
def generate_resizes
if is_video?
preview_file = generate_video_preview_for(video, width, height)
elsif is_ugoira?
preview_file = PixivUgoiraConverter.generate_preview(file)
sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data)
elsif is_image?
preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
if is_image? && image_width > Danbooru.config.large_image_width
generate_resize_for(Danbooru.config.large_image_width, nil, source_path)
if image_width > Danbooru.config.large_image_width
sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, nil, 90)
end
end
[preview_file, sample_file]
end
def generate_video_preview_for(width, height, output_path)
dimension_ratio = image_width.to_f / image_height
def generate_video_preview_for(video, width, height)
dimension_ratio = video.width.to_f / video.height
if dimension_ratio > 1
height = (width / dimension_ratio).to_i
else
width = (height * dimension_ratio).to_i
end
video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
FileUtils.chmod(0664, output_path)
end
def generate_resize_for(width, height, source_path, quality = 90)
unless File.exists?(source_path)
raise Error.new("file not found")
end
output_path = resized_file_path_for(width)
if is_image?
DanbooruImageResizer.resize(source_path, output_path, width, height, quality)
elsif is_ugoira?
if Delayed::Worker.delay_jobs
# by the time this runs we'll have moved source_path to md5_file_path
ugoira_service.generate_resizes(md5_file_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width))
else
ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width), false)
end
elsif is_video?
generate_video_preview_for(width, height, output_path)
end
output_file = Tempfile.new(binmode: true)
video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
output_file
end
end
@@ -287,10 +272,10 @@ class Upload < ApplicationRecord
if is_video?
[video.width, video.height]
elsif is_ugoira?
ugoira_service.calculate_dimensions(file_path)
ugoira_service.calculate_dimensions(file.path)
[ugoira_service.width, ugoira_service.height]
else
image_size = ImageSpec.new(file_path)
image_size = ImageSpec.new(file.path)
[image_size.width, image_size.height]
end
end
@@ -301,8 +286,8 @@ class Upload < ApplicationRecord
file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/
end
def file_header_to_file_ext(file_path)
case File.read(file_path, 16)
def file_header_to_file_ext(file)
case File.read(file.path, 16)
when /^\xff\xd8/n
"jpg"
when /^GIF87a/, /^GIF89a/
@@ -323,67 +308,18 @@ class Upload < ApplicationRecord
end
end
module FilePathMethods
def md5_file_path
prefix = Rails.env == "test" ? "test." : ""
"#{Rails.root}/public/data/#{prefix}#{md5}.#{file_ext}"
end
def resized_file_path_for(width)
prefix = Rails.env == "test" ? "test." : ""
case width
when Danbooru.config.small_image_width
"#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg"
when Danbooru.config.large_image_width
"#{Rails.root}/public/data/sample/#{prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}"
end
end
def large_file_ext
if is_ugoira?
"webm"
else
"jpg"
end
end
def temp_file_path
@temp_file_path ||= File.join(Rails.root, "tmp", "upload_#{Time.now.to_f}.#{Process.pid}")
end
end
module DownloaderMethods
# Determines whether the source is downloadable
def is_downloadable?
source =~ /^https?:\/\// && file_path.blank?
source =~ /^https?:\/\// && file.blank?
end
# Downloads the file to destination_path
def download_from_source(destination_path)
self.file_path = destination_path
download = Downloads::File.new(source, destination_path, :referer_url => referer_url)
download.download!
def download_from_source(source, referer_url)
download = Downloads::File.new(source, referer_url: referer_url)
file = download.download!
ugoira_service.load(download.data)
[download.downloaded_source, download.source]
end
end
module CgiFileMethods
def convert_cgi_file
return if file.blank? || file.size == 0
self.file_path = temp_file_path
if file.respond_to?(:tempfile) && file.tempfile
FileUtils.cp(file.tempfile.path, file_path)
else
File.open(file_path, 'wb') do |out|
out.write(file.read)
end
end
FileUtils.chmod(0664, file_path)
[download.downloaded_source, download.source, file]
end
end
@@ -422,7 +358,7 @@ class Upload < ApplicationRecord
module VideoMethods
def video
@video ||= FFMPEG::Movie.new(file_path)
@video ||= FFMPEG::Movie.new(file.path)
end
end
@@ -476,8 +412,6 @@ class Upload < ApplicationRecord
include DimensionMethods
include ContentTypeMethods
include DownloaderMethods
include FilePathMethods
include CgiFileMethods
include StatusMethods
include UploaderMethods
include VideoMethods