From c76463f34df003c8745a2d1410c1f1c208882356 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 18 Mar 2018 12:05:25 -0500 Subject: [PATCH] uploads: use storage manager to distribute files. Refactors the upload process to pass around temp files, rather than passing around file paths and directly writing output to the local filesystem. This way we can pass the storage manager the preview / sample / original temp files, so it can deal with storage itself. * Change Download::File#download! to return a temp file. * Change DanbooruImageResizer and PixivUgoiraConverter to accept/return temp files instead of file paths. * Change Upload#generate_resizes to return temp files for previews and samples. * Change Upload#generate_resizes to generate ugoira .webm samples synchronously instead of asynchronously. --- app/controllers/uploads_controller.rb | 2 +- app/logical/danbooru_image_resizer.rb | 9 +- app/logical/downloads/file.rb | 17 ++- app/logical/pixiv_ugoira_converter.rb | 29 +++-- app/logical/pixiv_ugoira_service.rb | 22 ---- app/models/post.rb | 15 +-- app/models/post_replacement.rb | 4 +- app/models/upload.rb | 174 ++++++++------------------ 8 files changed, 89 insertions(+), 183 deletions(-) diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index 055e7e660..55ace2f4e 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -6,7 +6,7 @@ class UploadsController < ApplicationController @upload = Upload.new @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first if params[:url] - download = Downloads::File.new(params[:url], ".") + download = Downloads::File.new(params[:url]) @normalized_url, _, _ = download.before_download(params[:url], {}) @post = find_post_by_url(@normalized_url) diff --git a/app/logical/danbooru_image_resizer.rb b/app/logical/danbooru_image_resizer.rb index 3871acc94..d3697fbc3 100644 --- a/app/logical/danbooru_image_resizer.rb +++ b/app/logical/danbooru_image_resizer.rb @@ -1,6 +1,6 @@ module DanbooruImageResizer - def resize(read_path, write_path, width, height, resize_quality = 90) - image = Magick::Image.read(read_path).first + def resize(file, width, height, resize_quality = 90) + image = Magick::Image.read(file.path).first geometry = "#{width}x>" if width == Danbooru.config.small_image_width @@ -17,14 +17,15 @@ module DanbooruImageResizer image = flatten(image, width, height) image.strip! - image.write(write_path) do + output_file = Tempfile.new(binmode: true) + image.write("jpeg:" + output_file.path) do self.quality = resize_quality # setting PlaneInterlace enables progressive encoding for JPEGs self.interlace = Magick::PlaneInterlace end image.destroy! - FileUtils.chmod(0664, write_path) + output_file end def flatten(image, width, height) diff --git a/app/logical/downloads/file.rb b/app/logical/downloads/file.rb index 911dc21df..ab1cf63ae 100644 --- a/app/logical/downloads/file.rb +++ b/app/logical/downloads/file.rb @@ -3,9 +3,9 @@ module Downloads class Error < Exception ; end attr_reader :data, :options - attr_accessor :source, :original_source, :downloaded_source, :file_path + attr_accessor :source, :original_source, :downloaded_source - def initialize(source, file_path, options = {}) + def initialize(source, options = {}) # source can potentially get rewritten in the course # of downloading a file, so check it again @source = source @@ -14,9 +14,6 @@ module Downloads # the URL actually downloaded after rewriting the original source. @downloaded_source = nil - # where to save the download - @file_path = file_path - # we sometimes need to capture data from the source page @data = {} @@ -35,12 +32,13 @@ module Downloads def download! url, headers, @data = before_download(@source, @data) - ::File.open(@file_path, "wb") do |out| - http_get_streaming(uncached_url(url, headers), out, headers) - end + output_file = Tempfile.new(binmode: true) + http_get_streaming(uncached_url(url, headers), output_file, headers) @downloaded_source = url @source = after_download(url) + + output_file end def before_download(url, datums) @@ -91,7 +89,8 @@ module Downloads end if res.success? - return + file.rewind + return file else raise Error.new("HTTP error code: #{res.code} #{res.message}") end diff --git a/app/logical/pixiv_ugoira_converter.rb b/app/logical/pixiv_ugoira_converter.rb index abe619656..791e14642 100644 --- a/app/logical/pixiv_ugoira_converter.rb +++ b/app/logical/pixiv_ugoira_converter.rb @@ -1,13 +1,9 @@ class PixivUgoiraConverter - def self.convert(source_path, output_path, preview_path, frame_data) - folder = Zip::File.new(source_path) - write_webm(folder, output_path, frame_data) - write_preview(folder, preview_path) - RemoteFileManager.new(output_path).distribute - RemoteFileManager.new(preview_path).distribute - end + def self.generate_webm(ugoira_file, frame_data) + folder = Zip::File.new(ugoira_file.path) + output_file = Tempfile.new(binmode: true) + write_path = output_file.path - def self.write_webm(folder, write_path, frame_data) Dir.mktmpdir do |tmpdir| FileUtils.mkdir_p("#{tmpdir}/images") folder.each_with_index do |file, i| @@ -64,14 +60,17 @@ class PixivUgoiraConverter return end end + + output_file end - def self.write_preview(folder, path) - Dir.mktmpdir do |tmpdir| - file = folder.first - temp_path = File.join(tmpdir, file.name) - file.extract(temp_path) - DanbooruImageResizer.resize(temp_path, path, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - end + def self.generate_preview(ugoira_file) + file = Tempfile.new(binmode: true) + zipfile = Zip::File.new(ugoira_file.path) + zipfile.entries.first.extract(file.path) { true } # 'true' means overwrite the existing tempfile. + + DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) + ensure + file.close! end end diff --git a/app/logical/pixiv_ugoira_service.rb b/app/logical/pixiv_ugoira_service.rb index 248428a8d..58536fa0d 100644 --- a/app/logical/pixiv_ugoira_service.rb +++ b/app/logical/pixiv_ugoira_service.rb @@ -1,32 +1,10 @@ class PixivUgoiraService attr_reader :width, :height, :frame_data, :content_type - def self.regen(post) - service = new() - service.load( - :is_ugoira => true, - :ugoira_frame_data => post.pixiv_ugoira_frame_data.data - ) - service.generate_resizes(post.file_path, post.large_file_path, post.preview_file_path, false) - end - def save_frame_data(post) PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id) end - def generate_resizes(source_path, output_path, preview_path, delay = true) - # Run this a bit in the future to give the upload process time to move the file - if delay - PixivUgoiraConverter.delay(:queue => Socket.gethostname, :run_at => 10.seconds.from_now, :priority => -1).convert(source_path, output_path, preview_path, @frame_data) - else - PixivUgoiraConverter.convert(source_path, output_path, preview_path, @frame_data) - end - - # since the resizes will be delayed, just touch the output file so the - # file distribution wont break - FileUtils.touch([output_path, preview_path]) - end - def calculate_dimensions(source_path) folder = Zip::File.new(source_path) tempfile = Tempfile.new("ugoira-dimensions") diff --git a/app/models/post.rb b/app/models/post.rb index 4528afe37..6b08b1d5b 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -130,17 +130,10 @@ class Post < ApplicationRecord Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true) end - def distribute_files - if Danbooru.config.build_file_url(self) =~ /^http/ - # this post is archived - RemoteFileManager.new(file_path).distribute_to_archive(Danbooru.config.build_file_url(self)) - RemoteFileManager.new(preview_file_path).distribute if has_preview? - RemoteFileManager.new(large_file_path).distribute_to_archive(Danbooru.config.build_large_file_url(self)) if has_large? - else - RemoteFileManager.new(file_path).distribute - RemoteFileManager.new(preview_file_path).distribute if has_preview? - RemoteFileManager.new(large_file_path).distribute if has_large? - end + def distribute_files(file, sample_file, preview_file) + storage_manager.store_file(file, self, :original) + storage_manager.store_file(sample_file, self, :large) if sample_file.present? + storage_manager.store_file(preview_file, self, :preview) if preview_file.present? end def file_path_prefix diff --git a/app/models/post_replacement.rb b/app/models/post_replacement.rb index aefe2715c..b6b3fd11b 100644 --- a/app/models/post_replacement.rb +++ b/app/models/post_replacement.rb @@ -24,6 +24,8 @@ class PostReplacement < ApplicationRecord end def process! + upload = nil + transaction do upload = Upload.create!( file: replacement_file, @@ -79,7 +81,7 @@ class PostReplacement < ApplicationRecord # point of no return: these things can't be rolled back, so we do them # only after the transaction successfully commits. - post.distribute_files + upload.distribute_files(post) post.update_iqdb_async end diff --git a/app/models/upload.rb b/app/models/upload.rb index a46984a9a..6b241c4d4 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -10,13 +10,11 @@ class Upload < ApplicationRecord belongs_to :uploader, :class_name => "User" belongs_to :post before_validation :initialize_uploader, :on => :create - before_create :convert_cgi_file - after_destroy :delete_temp_file validate :uploader_is_not_limited, :on => :create validate :file_or_source_is_present, :on => :create validate :rating_given attr_accessible :file, :image_width, :image_height, :file_ext, :md5, - :file_size, :as_pending, :source, :file_path, :content_type, :rating, + :file_size, :as_pending, :source, :rating, :tag_string, :status, :backtrace, :post_id, :md5_confirmation, :parent_id, :server, :artist_commentary_title, :artist_commentary_desc, :include_artist_commentary, @@ -101,31 +99,36 @@ class Upload < ApplicationRecord module ConversionMethods def process_upload - CurrentUser.scoped(uploader, uploader_ip_addr) do + begin update_attribute(:status, "processing") self.source = source.to_s.strip if is_downloadable? - self.downloaded_source, self.source = download_from_source(temp_file_path) + self.downloaded_source, self.source, self.file = download_from_source(source, referer_url) + else + self.file = self.file.tempfile end - self.file_ext = file_header_to_file_ext(file_path) + + self.file_ext = file_header_to_file_ext(file) + self.file_size = file.size + self.md5 = Digest::MD5.file(file.path).hexdigest + validate_file_content_type - calculate_hash(file_path) validate_md5_uniqueness validate_md5_confirmation validate_video_duration - calculate_file_size(file_path) + self.tag_string = "#{tag_string} #{automatic_tags}" self.image_width, self.image_height = calculate_dimensions - generate_resizes(file_path) - move_file + save end end def create_post_from_upload post = convert_to_post - post.distribute_files + distribute_files(post) + if post.save create_artist_commentary(post) if include_artist_commentary? ugoira_service.save_frame_data(post) if is_ugoira? @@ -138,6 +141,14 @@ class Upload < ApplicationRecord post end + def distribute_files(post) + preview_file, sample_file = generate_resizes + post.distribute_files(file, sample_file, preview_file) + ensure + preview_file.try(:close!) + sample_file.try(:close!) + end + def process!(force = false) @tries ||= 0 return if !force && status =~ /processing|completed|error/ @@ -157,9 +168,9 @@ class Upload < ApplicationRecord rescue Exception => x update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n")) nil - + ensure - delete_temp_file + file.try(:close!) end def ugoira_service @@ -194,23 +205,6 @@ class Upload < ApplicationRecord end module FileMethods - def delete_temp_file(path = nil) - FileUtils.rm_f(path || temp_file_path) - end - - def move_file - FileUtils.mv(file_path, md5_file_path) - end - - def calculate_file_size(source_path) - self.file_size = File.size(source_path) - end - - # Calculates the MD5 based on whatever is in temp_file_path - def calculate_hash(source_path) - self.md5 = Digest::MD5.file(source_path).hexdigest - end - def is_image? %w(jpg gif png).include?(file_ext) end @@ -232,52 +226,43 @@ class Upload < ApplicationRecord end def is_animated_gif? - file_ext == "gif" && Magick::Image.ping(file_path).length > 1 + file_ext == "gif" && Magick::Image.ping(file.path).length > 1 end def is_animated_png? - file_ext == "png" && APNGInspector.new(file_path).inspect!.animated? + file_ext == "png" && APNGInspector.new(file.path).inspect!.animated? end end module ResizerMethods - def generate_resizes(source_path) - generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85) + def generate_resizes + if is_video? + preview_file = generate_video_preview_for(video, width, height) + elsif is_ugoira? + preview_file = PixivUgoiraConverter.generate_preview(file) + sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data) + elsif is_image? + preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - if is_image? && image_width > Danbooru.config.large_image_width - generate_resize_for(Danbooru.config.large_image_width, nil, source_path) + if image_width > Danbooru.config.large_image_width + sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, nil, 90) + end end + + [preview_file, sample_file] end - def generate_video_preview_for(width, height, output_path) - dimension_ratio = image_width.to_f / image_height + def generate_video_preview_for(video, width, height) + dimension_ratio = video.width.to_f / video.height if dimension_ratio > 1 height = (width / dimension_ratio).to_i else width = (height * dimension_ratio).to_i end - video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) - FileUtils.chmod(0664, output_path) - end - def generate_resize_for(width, height, source_path, quality = 90) - unless File.exists?(source_path) - raise Error.new("file not found") - end - - output_path = resized_file_path_for(width) - if is_image? - DanbooruImageResizer.resize(source_path, output_path, width, height, quality) - elsif is_ugoira? - if Delayed::Worker.delay_jobs - # by the time this runs we'll have moved source_path to md5_file_path - ugoira_service.generate_resizes(md5_file_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width)) - else - ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width), false) - end - elsif is_video? - generate_video_preview_for(width, height, output_path) - end + output_file = Tempfile.new(binmode: true) + video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) + output_file end end @@ -287,10 +272,10 @@ class Upload < ApplicationRecord if is_video? [video.width, video.height] elsif is_ugoira? - ugoira_service.calculate_dimensions(file_path) + ugoira_service.calculate_dimensions(file.path) [ugoira_service.width, ugoira_service.height] else - image_size = ImageSpec.new(file_path) + image_size = ImageSpec.new(file.path) [image_size.width, image_size.height] end end @@ -301,8 +286,8 @@ class Upload < ApplicationRecord file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/ end - def file_header_to_file_ext(file_path) - case File.read(file_path, 16) + def file_header_to_file_ext(file) + case File.read(file.path, 16) when /^\xff\xd8/n "jpg" when /^GIF87a/, /^GIF89a/ @@ -323,67 +308,18 @@ class Upload < ApplicationRecord end end - module FilePathMethods - def md5_file_path - prefix = Rails.env == "test" ? "test." : "" - "#{Rails.root}/public/data/#{prefix}#{md5}.#{file_ext}" - end - - def resized_file_path_for(width) - prefix = Rails.env == "test" ? "test." : "" - - case width - when Danbooru.config.small_image_width - "#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg" - - when Danbooru.config.large_image_width - "#{Rails.root}/public/data/sample/#{prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}" - end - end - - def large_file_ext - if is_ugoira? - "webm" - else - "jpg" - end - end - - def temp_file_path - @temp_file_path ||= File.join(Rails.root, "tmp", "upload_#{Time.now.to_f}.#{Process.pid}") - end - end - module DownloaderMethods # Determines whether the source is downloadable def is_downloadable? - source =~ /^https?:\/\// && file_path.blank? + source =~ /^https?:\/\// && file.blank? end - # Downloads the file to destination_path - def download_from_source(destination_path) - self.file_path = destination_path - download = Downloads::File.new(source, destination_path, :referer_url => referer_url) - download.download! + def download_from_source(source, referer_url) + download = Downloads::File.new(source, referer_url: referer_url) + file = download.download! ugoira_service.load(download.data) - [download.downloaded_source, download.source] - end - end - module CgiFileMethods - def convert_cgi_file - return if file.blank? || file.size == 0 - - self.file_path = temp_file_path - - if file.respond_to?(:tempfile) && file.tempfile - FileUtils.cp(file.tempfile.path, file_path) - else - File.open(file_path, 'wb') do |out| - out.write(file.read) - end - end - FileUtils.chmod(0664, file_path) + [download.downloaded_source, download.source, file] end end @@ -422,7 +358,7 @@ class Upload < ApplicationRecord module VideoMethods def video - @video ||= FFMPEG::Movie.new(file_path) + @video ||= FFMPEG::Movie.new(file.path) end end @@ -476,8 +412,6 @@ class Upload < ApplicationRecord include DimensionMethods include ContentTypeMethods include DownloaderMethods - include FilePathMethods - include CgiFileMethods include StatusMethods include UploaderMethods include VideoMethods