diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index 055e7e660..55ace2f4e 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -6,7 +6,7 @@ class UploadsController < ApplicationController @upload = Upload.new @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first if params[:url] - download = Downloads::File.new(params[:url], ".") + download = Downloads::File.new(params[:url]) @normalized_url, _, _ = download.before_download(params[:url], {}) @post = find_post_by_url(@normalized_url) diff --git a/app/logical/danbooru_image_resizer.rb b/app/logical/danbooru_image_resizer.rb index 3871acc94..d3697fbc3 100644 --- a/app/logical/danbooru_image_resizer.rb +++ b/app/logical/danbooru_image_resizer.rb @@ -1,6 +1,6 @@ module DanbooruImageResizer - def resize(read_path, write_path, width, height, resize_quality = 90) - image = Magick::Image.read(read_path).first + def resize(file, width, height, resize_quality = 90) + image = Magick::Image.read(file.path).first geometry = "#{width}x>" if width == Danbooru.config.small_image_width @@ -17,14 +17,15 @@ module DanbooruImageResizer image = flatten(image, width, height) image.strip! - image.write(write_path) do + output_file = Tempfile.new(binmode: true) + image.write("jpeg:" + output_file.path) do self.quality = resize_quality # setting PlaneInterlace enables progressive encoding for JPEGs self.interlace = Magick::PlaneInterlace end image.destroy! - FileUtils.chmod(0664, write_path) + output_file end def flatten(image, width, height) diff --git a/app/logical/downloads/file.rb b/app/logical/downloads/file.rb index 911dc21df..ab1cf63ae 100644 --- a/app/logical/downloads/file.rb +++ b/app/logical/downloads/file.rb @@ -3,9 +3,9 @@ module Downloads class Error < Exception ; end attr_reader :data, :options - attr_accessor :source, :original_source, :downloaded_source, :file_path + attr_accessor :source, :original_source, :downloaded_source - def initialize(source, file_path, options = {}) + def initialize(source, options = {}) # source can potentially get rewritten in the course # of downloading a file, so check it again @source = source @@ -14,9 +14,6 @@ module Downloads # the URL actually downloaded after rewriting the original source. @downloaded_source = nil - # where to save the download - @file_path = file_path - # we sometimes need to capture data from the source page @data = {} @@ -35,12 +32,13 @@ module Downloads def download! url, headers, @data = before_download(@source, @data) - ::File.open(@file_path, "wb") do |out| - http_get_streaming(uncached_url(url, headers), out, headers) - end + output_file = Tempfile.new(binmode: true) + http_get_streaming(uncached_url(url, headers), output_file, headers) @downloaded_source = url @source = after_download(url) + + output_file end def before_download(url, datums) @@ -91,7 +89,8 @@ module Downloads end if res.success? - return + file.rewind + return file else raise Error.new("HTTP error code: #{res.code} #{res.message}") end diff --git a/app/logical/pixiv_ugoira_converter.rb b/app/logical/pixiv_ugoira_converter.rb index abe619656..791e14642 100644 --- a/app/logical/pixiv_ugoira_converter.rb +++ b/app/logical/pixiv_ugoira_converter.rb @@ -1,13 +1,9 @@ class PixivUgoiraConverter - def self.convert(source_path, output_path, preview_path, frame_data) - folder = Zip::File.new(source_path) - write_webm(folder, output_path, frame_data) - write_preview(folder, preview_path) - RemoteFileManager.new(output_path).distribute - RemoteFileManager.new(preview_path).distribute - end + def self.generate_webm(ugoira_file, frame_data) + folder = Zip::File.new(ugoira_file.path) + output_file = Tempfile.new(binmode: true) + write_path = output_file.path - def self.write_webm(folder, write_path, frame_data) Dir.mktmpdir do |tmpdir| FileUtils.mkdir_p("#{tmpdir}/images") folder.each_with_index do |file, i| @@ -64,14 +60,17 @@ class PixivUgoiraConverter return end end + + output_file end - def self.write_preview(folder, path) - Dir.mktmpdir do |tmpdir| - file = folder.first - temp_path = File.join(tmpdir, file.name) - file.extract(temp_path) - DanbooruImageResizer.resize(temp_path, path, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - end + def self.generate_preview(ugoira_file) + file = Tempfile.new(binmode: true) + zipfile = Zip::File.new(ugoira_file.path) + zipfile.entries.first.extract(file.path) { true } # 'true' means overwrite the existing tempfile. + + DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) + ensure + file.close! end end diff --git a/app/logical/pixiv_ugoira_service.rb b/app/logical/pixiv_ugoira_service.rb index 248428a8d..58536fa0d 100644 --- a/app/logical/pixiv_ugoira_service.rb +++ b/app/logical/pixiv_ugoira_service.rb @@ -1,32 +1,10 @@ class PixivUgoiraService attr_reader :width, :height, :frame_data, :content_type - def self.regen(post) - service = new() - service.load( - :is_ugoira => true, - :ugoira_frame_data => post.pixiv_ugoira_frame_data.data - ) - service.generate_resizes(post.file_path, post.large_file_path, post.preview_file_path, false) - end - def save_frame_data(post) PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id) end - def generate_resizes(source_path, output_path, preview_path, delay = true) - # Run this a bit in the future to give the upload process time to move the file - if delay - PixivUgoiraConverter.delay(:queue => Socket.gethostname, :run_at => 10.seconds.from_now, :priority => -1).convert(source_path, output_path, preview_path, @frame_data) - else - PixivUgoiraConverter.convert(source_path, output_path, preview_path, @frame_data) - end - - # since the resizes will be delayed, just touch the output file so the - # file distribution wont break - FileUtils.touch([output_path, preview_path]) - end - def calculate_dimensions(source_path) folder = Zip::File.new(source_path) tempfile = Tempfile.new("ugoira-dimensions") diff --git a/app/models/post.rb b/app/models/post.rb index 4528afe37..6b08b1d5b 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -130,17 +130,10 @@ class Post < ApplicationRecord Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true) end - def distribute_files - if Danbooru.config.build_file_url(self) =~ /^http/ - # this post is archived - RemoteFileManager.new(file_path).distribute_to_archive(Danbooru.config.build_file_url(self)) - RemoteFileManager.new(preview_file_path).distribute if has_preview? - RemoteFileManager.new(large_file_path).distribute_to_archive(Danbooru.config.build_large_file_url(self)) if has_large? - else - RemoteFileManager.new(file_path).distribute - RemoteFileManager.new(preview_file_path).distribute if has_preview? - RemoteFileManager.new(large_file_path).distribute if has_large? - end + def distribute_files(file, sample_file, preview_file) + storage_manager.store_file(file, self, :original) + storage_manager.store_file(sample_file, self, :large) if sample_file.present? + storage_manager.store_file(preview_file, self, :preview) if preview_file.present? end def file_path_prefix diff --git a/app/models/post_replacement.rb b/app/models/post_replacement.rb index aefe2715c..b6b3fd11b 100644 --- a/app/models/post_replacement.rb +++ b/app/models/post_replacement.rb @@ -24,6 +24,8 @@ class PostReplacement < ApplicationRecord end def process! + upload = nil + transaction do upload = Upload.create!( file: replacement_file, @@ -79,7 +81,7 @@ class PostReplacement < ApplicationRecord # point of no return: these things can't be rolled back, so we do them # only after the transaction successfully commits. - post.distribute_files + upload.distribute_files(post) post.update_iqdb_async end diff --git a/app/models/upload.rb b/app/models/upload.rb index a46984a9a..6b241c4d4 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -10,13 +10,11 @@ class Upload < ApplicationRecord belongs_to :uploader, :class_name => "User" belongs_to :post before_validation :initialize_uploader, :on => :create - before_create :convert_cgi_file - after_destroy :delete_temp_file validate :uploader_is_not_limited, :on => :create validate :file_or_source_is_present, :on => :create validate :rating_given attr_accessible :file, :image_width, :image_height, :file_ext, :md5, - :file_size, :as_pending, :source, :file_path, :content_type, :rating, + :file_size, :as_pending, :source, :rating, :tag_string, :status, :backtrace, :post_id, :md5_confirmation, :parent_id, :server, :artist_commentary_title, :artist_commentary_desc, :include_artist_commentary, @@ -101,31 +99,36 @@ class Upload < ApplicationRecord module ConversionMethods def process_upload - CurrentUser.scoped(uploader, uploader_ip_addr) do + begin update_attribute(:status, "processing") self.source = source.to_s.strip if is_downloadable? - self.downloaded_source, self.source = download_from_source(temp_file_path) + self.downloaded_source, self.source, self.file = download_from_source(source, referer_url) + else + self.file = self.file.tempfile end - self.file_ext = file_header_to_file_ext(file_path) + + self.file_ext = file_header_to_file_ext(file) + self.file_size = file.size + self.md5 = Digest::MD5.file(file.path).hexdigest + validate_file_content_type - calculate_hash(file_path) validate_md5_uniqueness validate_md5_confirmation validate_video_duration - calculate_file_size(file_path) + self.tag_string = "#{tag_string} #{automatic_tags}" self.image_width, self.image_height = calculate_dimensions - generate_resizes(file_path) - move_file + save end end def create_post_from_upload post = convert_to_post - post.distribute_files + distribute_files(post) + if post.save create_artist_commentary(post) if include_artist_commentary? ugoira_service.save_frame_data(post) if is_ugoira? @@ -138,6 +141,14 @@ class Upload < ApplicationRecord post end + def distribute_files(post) + preview_file, sample_file = generate_resizes + post.distribute_files(file, sample_file, preview_file) + ensure + preview_file.try(:close!) + sample_file.try(:close!) + end + def process!(force = false) @tries ||= 0 return if !force && status =~ /processing|completed|error/ @@ -157,9 +168,9 @@ class Upload < ApplicationRecord rescue Exception => x update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n")) nil - + ensure - delete_temp_file + file.try(:close!) end def ugoira_service @@ -194,23 +205,6 @@ class Upload < ApplicationRecord end module FileMethods - def delete_temp_file(path = nil) - FileUtils.rm_f(path || temp_file_path) - end - - def move_file - FileUtils.mv(file_path, md5_file_path) - end - - def calculate_file_size(source_path) - self.file_size = File.size(source_path) - end - - # Calculates the MD5 based on whatever is in temp_file_path - def calculate_hash(source_path) - self.md5 = Digest::MD5.file(source_path).hexdigest - end - def is_image? %w(jpg gif png).include?(file_ext) end @@ -232,52 +226,43 @@ class Upload < ApplicationRecord end def is_animated_gif? - file_ext == "gif" && Magick::Image.ping(file_path).length > 1 + file_ext == "gif" && Magick::Image.ping(file.path).length > 1 end def is_animated_png? - file_ext == "png" && APNGInspector.new(file_path).inspect!.animated? + file_ext == "png" && APNGInspector.new(file.path).inspect!.animated? end end module ResizerMethods - def generate_resizes(source_path) - generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85) + def generate_resizes + if is_video? + preview_file = generate_video_preview_for(video, width, height) + elsif is_ugoira? + preview_file = PixivUgoiraConverter.generate_preview(file) + sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data) + elsif is_image? + preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - if is_image? && image_width > Danbooru.config.large_image_width - generate_resize_for(Danbooru.config.large_image_width, nil, source_path) + if image_width > Danbooru.config.large_image_width + sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, nil, 90) + end end + + [preview_file, sample_file] end - def generate_video_preview_for(width, height, output_path) - dimension_ratio = image_width.to_f / image_height + def generate_video_preview_for(video, width, height) + dimension_ratio = video.width.to_f / video.height if dimension_ratio > 1 height = (width / dimension_ratio).to_i else width = (height * dimension_ratio).to_i end - video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) - FileUtils.chmod(0664, output_path) - end - def generate_resize_for(width, height, source_path, quality = 90) - unless File.exists?(source_path) - raise Error.new("file not found") - end - - output_path = resized_file_path_for(width) - if is_image? - DanbooruImageResizer.resize(source_path, output_path, width, height, quality) - elsif is_ugoira? - if Delayed::Worker.delay_jobs - # by the time this runs we'll have moved source_path to md5_file_path - ugoira_service.generate_resizes(md5_file_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width)) - else - ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width), false) - end - elsif is_video? - generate_video_preview_for(width, height, output_path) - end + output_file = Tempfile.new(binmode: true) + video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) + output_file end end @@ -287,10 +272,10 @@ class Upload < ApplicationRecord if is_video? [video.width, video.height] elsif is_ugoira? - ugoira_service.calculate_dimensions(file_path) + ugoira_service.calculate_dimensions(file.path) [ugoira_service.width, ugoira_service.height] else - image_size = ImageSpec.new(file_path) + image_size = ImageSpec.new(file.path) [image_size.width, image_size.height] end end @@ -301,8 +286,8 @@ class Upload < ApplicationRecord file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/ end - def file_header_to_file_ext(file_path) - case File.read(file_path, 16) + def file_header_to_file_ext(file) + case File.read(file.path, 16) when /^\xff\xd8/n "jpg" when /^GIF87a/, /^GIF89a/ @@ -323,67 +308,18 @@ class Upload < ApplicationRecord end end - module FilePathMethods - def md5_file_path - prefix = Rails.env == "test" ? "test." : "" - "#{Rails.root}/public/data/#{prefix}#{md5}.#{file_ext}" - end - - def resized_file_path_for(width) - prefix = Rails.env == "test" ? "test." : "" - - case width - when Danbooru.config.small_image_width - "#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg" - - when Danbooru.config.large_image_width - "#{Rails.root}/public/data/sample/#{prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}" - end - end - - def large_file_ext - if is_ugoira? - "webm" - else - "jpg" - end - end - - def temp_file_path - @temp_file_path ||= File.join(Rails.root, "tmp", "upload_#{Time.now.to_f}.#{Process.pid}") - end - end - module DownloaderMethods # Determines whether the source is downloadable def is_downloadable? - source =~ /^https?:\/\// && file_path.blank? + source =~ /^https?:\/\// && file.blank? end - # Downloads the file to destination_path - def download_from_source(destination_path) - self.file_path = destination_path - download = Downloads::File.new(source, destination_path, :referer_url => referer_url) - download.download! + def download_from_source(source, referer_url) + download = Downloads::File.new(source, referer_url: referer_url) + file = download.download! ugoira_service.load(download.data) - [download.downloaded_source, download.source] - end - end - module CgiFileMethods - def convert_cgi_file - return if file.blank? || file.size == 0 - - self.file_path = temp_file_path - - if file.respond_to?(:tempfile) && file.tempfile - FileUtils.cp(file.tempfile.path, file_path) - else - File.open(file_path, 'wb') do |out| - out.write(file.read) - end - end - FileUtils.chmod(0664, file_path) + [download.downloaded_source, download.source, file] end end @@ -422,7 +358,7 @@ class Upload < ApplicationRecord module VideoMethods def video - @video ||= FFMPEG::Movie.new(file_path) + @video ||= FFMPEG::Movie.new(file.path) end end @@ -476,8 +412,6 @@ class Upload < ApplicationRecord include DimensionMethods include ContentTypeMethods include DownloaderMethods - include FilePathMethods - include CgiFileMethods include StatusMethods include UploaderMethods include VideoMethods