jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

uploads: use storage manager to distribute files.

Browse Source 
Refactors the upload process to pass around temp files, rather than
passing around file paths and directly writing output to the local
filesystem. This way we can pass the storage manager the preview /
sample / original temp files, so it can deal with storage itself.

* Change Download::File#download! to return a temp file.

* Change DanbooruImageResizer and PixivUgoiraConverter to accept/return
  temp files instead of file paths.

* Change Upload#generate_resizes to return temp files for previews and samples.

* Change Upload#generate_resizes to generate ugoira .webm samples
  synchronously instead of asynchronously.
This commit is contained in:
evazion
2018-03-18 12:05:25 -05:00
parent 60dcfbfbdd
commit c76463f34d
8 changed files with 89 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/uploads_controller.rb
View File

@@ -6,7 +6,7 @@ class UploadsController < ApplicationController
@upload = Upload.new @upload = Upload.new
@upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
if params[:url] if params[:url]
download = Downloads::File.new(params[:url], ".") download = Downloads::File.new(params[:url])
@normalized_url, _, _ = download.before_download(params[:url], {}) @normalized_url, _, _ = download.before_download(params[:url], {})
@post = find_post_by_url(@normalized_url) @post = find_post_by_url(@normalized_url)

9
app/logical/danbooru_image_resizer.rb
View File

@@ -1,6 +1,6 @@
module DanbooruImageResizer module DanbooruImageResizer
def resize(read_path, write_path, width, height, resize_quality = 90) def resize(file, width, height, resize_quality = 90)
image = Magick::Image.read(read_path).first image = Magick::Image.read(file.path).first
geometry = "#{width}x>" geometry = "#{width}x>"
if width == Danbooru.config.small_image_width if width == Danbooru.config.small_image_width
@@ -17,14 +17,15 @@ module DanbooruImageResizer
image = flatten(image, width, height) image = flatten(image, width, height)
image.strip! image.strip!
image.write(write_path) do output_file = Tempfile.new(binmode: true)
image.write("jpeg:" + output_file.path) do
self.quality = resize_quality self.quality = resize_quality
# setting PlaneInterlace enables progressive encoding for JPEGs # setting PlaneInterlace enables progressive encoding for JPEGs
self.interlace = Magick::PlaneInterlace self.interlace = Magick::PlaneInterlace
end end
image.destroy! image.destroy!
FileUtils.chmod(0664, write_path) output_file
end end
def flatten(image, width, height) def flatten(image, width, height)

17
app/logical/downloads/file.rb
View File

@@ -3,9 +3,9 @@ module Downloads
class Error < Exception ; end class Error < Exception ; end
attr_reader :data, :options attr_reader :data, :options
attr_accessor :source, :original_source, :downloaded_source, :file_path attr_accessor :source, :original_source, :downloaded_source
def initialize(source, file_path, options = {}) def initialize(source, options = {})
# source can potentially get rewritten in the course # source can potentially get rewritten in the course
# of downloading a file, so check it again # of downloading a file, so check it again
@source = source @source = source
@@ -14,9 +14,6 @@ module Downloads
# the URL actually downloaded after rewriting the original source. # the URL actually downloaded after rewriting the original source.
@downloaded_source = nil @downloaded_source = nil
# where to save the download
@file_path = file_path
# we sometimes need to capture data from the source page # we sometimes need to capture data from the source page
@data = {} @data = {}
@@ -35,12 +32,13 @@ module Downloads
def download! def download!
url, headers, @data = before_download(@source, @data) url, headers, @data = before_download(@source, @data)
::File.open(@file_path, "wb") do |out| output_file = Tempfile.new(binmode: true)
http_get_streaming(uncached_url(url, headers), out, headers) http_get_streaming(uncached_url(url, headers), output_file, headers)
end
@downloaded_source = url @downloaded_source = url
@source = after_download(url) @source = after_download(url)
output_file
end end
def before_download(url, datums) def before_download(url, datums)
@@ -91,7 +89,8 @@ module Downloads
end end
if res.success? if res.success?
return file.rewind
return file
else else
raise Error.new("HTTP error code: #{res.code} #{res.message}") raise Error.new("HTTP error code: #{res.code} #{res.message}")
end end

29
app/logical/pixiv_ugoira_converter.rb
View File

@@ -1,13 +1,9 @@
class PixivUgoiraConverter class PixivUgoiraConverter
def self.convert(source_path, output_path, preview_path, frame_data) def self.generate_webm(ugoira_file, frame_data)
folder = Zip::File.new(source_path) folder = Zip::File.new(ugoira_file.path)
write_webm(folder, output_path, frame_data) output_file = Tempfile.new(binmode: true)
write_preview(folder, preview_path) write_path = output_file.path
RemoteFileManager.new(output_path).distribute
RemoteFileManager.new(preview_path).distribute
end
def self.write_webm(folder, write_path, frame_data)
Dir.mktmpdir do |tmpdir| Dir.mktmpdir do |tmpdir|
FileUtils.mkdir_p("#{tmpdir}/images") FileUtils.mkdir_p("#{tmpdir}/images")
folder.each_with_index do |file, i| folder.each_with_index do |file, i|
@@ -64,14 +60,17 @@ class PixivUgoiraConverter
return return
end end
end end
output_file
end end
def self.write_preview(folder, path) def self.generate_preview(ugoira_file)
Dir.mktmpdir do |tmpdir| file = Tempfile.new(binmode: true)
file = folder.first zipfile = Zip::File.new(ugoira_file.path)
temp_path = File.join(tmpdir, file.name) zipfile.entries.first.extract(file.path) { true } # 'true' means overwrite the existing tempfile.
file.extract(temp_path)
DanbooruImageResizer.resize(temp_path, path, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
end ensure
file.close!
end end
end end

22
app/logical/pixiv_ugoira_service.rb
View File

@@ -1,32 +1,10 @@
class PixivUgoiraService class PixivUgoiraService
attr_reader :width, :height, :frame_data, :content_type attr_reader :width, :height, :frame_data, :content_type
def self.regen(post)
service = new()
service.load(
:is_ugoira => true,
:ugoira_frame_data => post.pixiv_ugoira_frame_data.data
)
service.generate_resizes(post.file_path, post.large_file_path, post.preview_file_path, false)
end
def save_frame_data(post) def save_frame_data(post)
PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id) PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id)
end end
def generate_resizes(source_path, output_path, preview_path, delay = true)
# Run this a bit in the future to give the upload process time to move the file
if delay
PixivUgoiraConverter.delay(:queue => Socket.gethostname, :run_at => 10.seconds.from_now, :priority => -1).convert(source_path, output_path, preview_path, @frame_data)
else
PixivUgoiraConverter.convert(source_path, output_path, preview_path, @frame_data)
end
# since the resizes will be delayed, just touch the output file so the
# file distribution wont break
FileUtils.touch([output_path, preview_path])
end
def calculate_dimensions(source_path) def calculate_dimensions(source_path)
folder = Zip::File.new(source_path) folder = Zip::File.new(source_path)
tempfile = Tempfile.new("ugoira-dimensions") tempfile = Tempfile.new("ugoira-dimensions")

15
app/models/post.rb
View File

@@ -130,17 +130,10 @@ class Post < ApplicationRecord
Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true) Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true)
end end
def distribute_files def distribute_files(file, sample_file, preview_file)
if Danbooru.config.build_file_url(self) =~ /^http/ storage_manager.store_file(file, self, :original)
# this post is archived storage_manager.store_file(sample_file, self, :large) if sample_file.present?
RemoteFileManager.new(file_path).distribute_to_archive(Danbooru.config.build_file_url(self)) storage_manager.store_file(preview_file, self, :preview) if preview_file.present?
RemoteFileManager.new(preview_file_path).distribute if has_preview?
RemoteFileManager.new(large_file_path).distribute_to_archive(Danbooru.config.build_large_file_url(self)) if has_large?
else
RemoteFileManager.new(file_path).distribute
RemoteFileManager.new(preview_file_path).distribute if has_preview?
RemoteFileManager.new(large_file_path).distribute if has_large?
end
end end
def file_path_prefix def file_path_prefix

4
app/models/post_replacement.rb
View File

@@ -24,6 +24,8 @@ class PostReplacement < ApplicationRecord
end end
def process! def process!
upload = nil
transaction do transaction do
upload = Upload.create!( upload = Upload.create!(
file: replacement_file, file: replacement_file,
@@ -79,7 +81,7 @@ class PostReplacement < ApplicationRecord
# point of no return: these things can't be rolled back, so we do them # point of no return: these things can't be rolled back, so we do them
# only after the transaction successfully commits. # only after the transaction successfully commits.
post.distribute_files upload.distribute_files(post)
post.update_iqdb_async post.update_iqdb_async
end end

174
app/models/upload.rb
View File

@@ -10,13 +10,11 @@ class Upload < ApplicationRecord
belongs_to :uploader, :class_name => "User" belongs_to :uploader, :class_name => "User"
belongs_to :post belongs_to :post
before_validation :initialize_uploader, :on => :create before_validation :initialize_uploader, :on => :create
before_create :convert_cgi_file
after_destroy :delete_temp_file
validate :uploader_is_not_limited, :on => :create validate :uploader_is_not_limited, :on => :create
validate :file_or_source_is_present, :on => :create validate :file_or_source_is_present, :on => :create
validate :rating_given validate :rating_given
attr_accessible :file, :image_width, :image_height, :file_ext, :md5, attr_accessible :file, :image_width, :image_height, :file_ext, :md5,
:file_size, :as_pending, :source, :file_path, :content_type, :rating, :file_size, :as_pending, :source, :rating,
:tag_string, :status, :backtrace, :post_id, :md5_confirmation, :tag_string, :status, :backtrace, :post_id, :md5_confirmation,
:parent_id, :server, :artist_commentary_title, :parent_id, :server, :artist_commentary_title,
:artist_commentary_desc, :include_artist_commentary, :artist_commentary_desc, :include_artist_commentary,
@@ -101,31 +99,36 @@ class Upload < ApplicationRecord
module ConversionMethods module ConversionMethods
def process_upload def process_upload
CurrentUser.scoped(uploader, uploader_ip_addr) do begin
update_attribute(:status, "processing") update_attribute(:status, "processing")
self.source = source.to_s.strip self.source = source.to_s.strip
if is_downloadable? if is_downloadable?
self.downloaded_source, self.source = download_from_source(temp_file_path) self.downloaded_source, self.source, self.file = download_from_source(source, referer_url)
else
self.file = self.file.tempfile
end end
self.file_ext = file_header_to_file_ext(file_path)
self.file_ext = file_header_to_file_ext(file)
self.file_size = file.size
self.md5 = Digest::MD5.file(file.path).hexdigest
validate_file_content_type validate_file_content_type
calculate_hash(file_path)
validate_md5_uniqueness validate_md5_uniqueness
validate_md5_confirmation validate_md5_confirmation
validate_video_duration validate_video_duration
calculate_file_size(file_path)
self.tag_string = "#{tag_string} #{automatic_tags}" self.tag_string = "#{tag_string} #{automatic_tags}"
self.image_width, self.image_height = calculate_dimensions self.image_width, self.image_height = calculate_dimensions
generate_resizes(file_path)
move_file
save save
end end
end end
def create_post_from_upload def create_post_from_upload
post = convert_to_post post = convert_to_post
post.distribute_files distribute_files(post)
if post.save if post.save
create_artist_commentary(post) if include_artist_commentary? create_artist_commentary(post) if include_artist_commentary?
ugoira_service.save_frame_data(post) if is_ugoira? ugoira_service.save_frame_data(post) if is_ugoira?
@@ -138,6 +141,14 @@ class Upload < ApplicationRecord
post post
end end
def distribute_files(post)
preview_file, sample_file = generate_resizes
post.distribute_files(file, sample_file, preview_file)
ensure
preview_file.try(:close!)
sample_file.try(:close!)
end
def process!(force = false) def process!(force = false)
@tries ||= 0 @tries ||= 0
return if !force && status =~ /processing|completed|error/ return if !force && status =~ /processing|completed|error/
@@ -157,9 +168,9 @@ class Upload < ApplicationRecord
rescue Exception => x rescue Exception => x
update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n")) update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
nil nil
ensure ensure
delete_temp_file file.try(:close!)
end end
def ugoira_service def ugoira_service
@@ -194,23 +205,6 @@ class Upload < ApplicationRecord
end end
module FileMethods module FileMethods
def delete_temp_file(path = nil)
FileUtils.rm_f(path || temp_file_path)
end
def move_file
FileUtils.mv(file_path, md5_file_path)
end
def calculate_file_size(source_path)
self.file_size = File.size(source_path)
end
# Calculates the MD5 based on whatever is in temp_file_path
def calculate_hash(source_path)
self.md5 = Digest::MD5.file(source_path).hexdigest
end
def is_image? def is_image?
%w(jpg gif png).include?(file_ext) %w(jpg gif png).include?(file_ext)
end end
@@ -232,52 +226,43 @@ class Upload < ApplicationRecord
end end
def is_animated_gif? def is_animated_gif?
file_ext == "gif" && Magick::Image.ping(file_path).length > 1 file_ext == "gif" && Magick::Image.ping(file.path).length > 1
end end
def is_animated_png? def is_animated_png?
file_ext == "png" && APNGInspector.new(file_path).inspect!.animated? file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
end end
end end
module ResizerMethods module ResizerMethods
def generate_resizes(source_path) def generate_resizes
generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85) if is_video?
preview_file = generate_video_preview_for(video, width, height)
elsif is_ugoira?
preview_file = PixivUgoiraConverter.generate_preview(file)
sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data)
elsif is_image?
preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
if is_image? && image_width > Danbooru.config.large_image_width if image_width > Danbooru.config.large_image_width
generate_resize_for(Danbooru.config.large_image_width, nil, source_path) sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, nil, 90)
end
end end
[preview_file, sample_file]
end end
def generate_video_preview_for(width, height, output_path) def generate_video_preview_for(video, width, height)
dimension_ratio = image_width.to_f / image_height dimension_ratio = video.width.to_f / video.height
if dimension_ratio > 1 if dimension_ratio > 1
height = (width / dimension_ratio).to_i height = (width / dimension_ratio).to_i
else else
width = (height * dimension_ratio).to_i width = (height * dimension_ratio).to_i
end end
video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
FileUtils.chmod(0664, output_path)
end
def generate_resize_for(width, height, source_path, quality = 90) output_file = Tempfile.new(binmode: true)
unless File.exists?(source_path) video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
raise Error.new("file not found") output_file
end
output_path = resized_file_path_for(width)
if is_image?
DanbooruImageResizer.resize(source_path, output_path, width, height, quality)
elsif is_ugoira?
if Delayed::Worker.delay_jobs
# by the time this runs we'll have moved source_path to md5_file_path
ugoira_service.generate_resizes(md5_file_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width))
else
ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width), false)
end
elsif is_video?
generate_video_preview_for(width, height, output_path)
end
end end
end end
@@ -287,10 +272,10 @@ class Upload < ApplicationRecord
if is_video? if is_video?
[video.width, video.height] [video.width, video.height]
elsif is_ugoira? elsif is_ugoira?
ugoira_service.calculate_dimensions(file_path) ugoira_service.calculate_dimensions(file.path)
[ugoira_service.width, ugoira_service.height] [ugoira_service.width, ugoira_service.height]
else else
image_size = ImageSpec.new(file_path) image_size = ImageSpec.new(file.path)
[image_size.width, image_size.height] [image_size.width, image_size.height]
end end
end end
@@ -301,8 +286,8 @@ class Upload < ApplicationRecord
file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/ file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/
end end
def file_header_to_file_ext(file_path) def file_header_to_file_ext(file)
case File.read(file_path, 16) case File.read(file.path, 16)
when /^\xff\xd8/n when /^\xff\xd8/n
"jpg" "jpg"
when /^GIF87a/, /^GIF89a/ when /^GIF87a/, /^GIF89a/
@@ -323,67 +308,18 @@ class Upload < ApplicationRecord
end end
end end
module FilePathMethods
def md5_file_path
prefix = Rails.env == "test" ? "test." : ""
"#{Rails.root}/public/data/#{prefix}#{md5}.#{file_ext}"
end
def resized_file_path_for(width)
prefix = Rails.env == "test" ? "test." : ""
case width
when Danbooru.config.small_image_width
"#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg"
when Danbooru.config.large_image_width
"#{Rails.root}/public/data/sample/#{prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}"
end
end
def large_file_ext
if is_ugoira?
"webm"
else
"jpg"
end
end
def temp_file_path
@temp_file_path ||= File.join(Rails.root, "tmp", "upload_#{Time.now.to_f}.#{Process.pid}")
end
end
module DownloaderMethods module DownloaderMethods
# Determines whether the source is downloadable # Determines whether the source is downloadable
def is_downloadable? def is_downloadable?
source =~ /^https?:\/\// && file_path.blank? source =~ /^https?:\/\// && file.blank?
end end
# Downloads the file to destination_path def download_from_source(source, referer_url)
def download_from_source(destination_path) download = Downloads::File.new(source, referer_url: referer_url)
self.file_path = destination_path file = download.download!
download = Downloads::File.new(source, destination_path, :referer_url => referer_url)
download.download!
ugoira_service.load(download.data) ugoira_service.load(download.data)
[download.downloaded_source, download.source]
end
end
module CgiFileMethods [download.downloaded_source, download.source, file]
def convert_cgi_file
return if file.blank? || file.size == 0
self.file_path = temp_file_path
if file.respond_to?(:tempfile) && file.tempfile
FileUtils.cp(file.tempfile.path, file_path)
else
File.open(file_path, 'wb') do |out|
out.write(file.read)
end
end
FileUtils.chmod(0664, file_path)
end end
end end
@@ -422,7 +358,7 @@ class Upload < ApplicationRecord
module VideoMethods module VideoMethods
def video def video
@video ||= FFMPEG::Movie.new(file_path) @video ||= FFMPEG::Movie.new(file.path)
end end
end end
@@ -476,8 +412,6 @@ class Upload < ApplicationRecord
include DimensionMethods include DimensionMethods
include ContentTypeMethods include ContentTypeMethods
include DownloaderMethods include DownloaderMethods
include FilePathMethods
include CgiFileMethods
include StatusMethods include StatusMethods
include UploaderMethods include UploaderMethods
include VideoMethods include VideoMethods