diff --git a/app/controllers/iqdb_queries_controller.rb b/app/controllers/iqdb_queries_controller.rb index b9aae52c0..62a9b93a5 100644 --- a/app/controllers/iqdb_queries_controller.rb +++ b/app/controllers/iqdb_queries_controller.rb @@ -49,7 +49,7 @@ protected def create_by_post @post = Post.find(params[:post_id]) - @download = Iqdb::Download.new(@post.complete_preview_file_url) + @download = Iqdb::Download.new(@post.preview_file_url) @download.find_similar @results = @download.matches end diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index 055e7e660..55ace2f4e 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -6,7 +6,7 @@ class UploadsController < ApplicationController @upload = Upload.new @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first if params[:url] - download = Downloads::File.new(params[:url], ".") + download = Downloads::File.new(params[:url]) @normalized_url, _, _ = download.before_download(params[:url], {}) @post = find_post_by_url(@normalized_url) diff --git a/app/logical/apng_inspector.rb b/app/logical/apng_inspector.rb index 30ce02676..dffad8081 100644 --- a/app/logical/apng_inspector.rb +++ b/app/logical/apng_inspector.rb @@ -86,7 +86,7 @@ class APNGInspector end @corrupted = !read_success || actl_corrupted - return !@corrupted + self end def corrupted? @@ -109,4 +109,4 @@ class APNGInspector return framedata.unpack("N".freeze)[0] end -end \ No newline at end of file +end diff --git a/app/logical/backup_service.rb b/app/logical/backup_service.rb deleted file mode 100644 index c34cdc339..000000000 --- a/app/logical/backup_service.rb +++ /dev/null @@ -1,9 +0,0 @@ -class BackupService - def backup(file_path, options = {}) - raise NotImplementedError.new("#{self.class}.backup not implemented") - end - - def delete(file_path, options = {}) - raise NotImplementedError.new("#{self.class}.delete not implemented") - end -end diff --git a/app/logical/danbooru_image_resizer.rb b/app/logical/danbooru_image_resizer.rb index 3871acc94..d3697fbc3 100644 --- a/app/logical/danbooru_image_resizer.rb +++ b/app/logical/danbooru_image_resizer.rb @@ -1,6 +1,6 @@ module DanbooruImageResizer - def resize(read_path, write_path, width, height, resize_quality = 90) - image = Magick::Image.read(read_path).first + def resize(file, width, height, resize_quality = 90) + image = Magick::Image.read(file.path).first geometry = "#{width}x>" if width == Danbooru.config.small_image_width @@ -17,14 +17,15 @@ module DanbooruImageResizer image = flatten(image, width, height) image.strip! - image.write(write_path) do + output_file = Tempfile.new(binmode: true) + image.write("jpeg:" + output_file.path) do self.quality = resize_quality # setting PlaneInterlace enables progressive encoding for JPEGs self.interlace = Magick::PlaneInterlace end image.destroy! - FileUtils.chmod(0664, write_path) + output_file end def flatten(image, width, height) diff --git a/app/logical/downloads/file.rb b/app/logical/downloads/file.rb index 911dc21df..ab1cf63ae 100644 --- a/app/logical/downloads/file.rb +++ b/app/logical/downloads/file.rb @@ -3,9 +3,9 @@ module Downloads class Error < Exception ; end attr_reader :data, :options - attr_accessor :source, :original_source, :downloaded_source, :file_path + attr_accessor :source, :original_source, :downloaded_source - def initialize(source, file_path, options = {}) + def initialize(source, options = {}) # source can potentially get rewritten in the course # of downloading a file, so check it again @source = source @@ -14,9 +14,6 @@ module Downloads # the URL actually downloaded after rewriting the original source. @downloaded_source = nil - # where to save the download - @file_path = file_path - # we sometimes need to capture data from the source page @data = {} @@ -35,12 +32,13 @@ module Downloads def download! url, headers, @data = before_download(@source, @data) - ::File.open(@file_path, "wb") do |out| - http_get_streaming(uncached_url(url, headers), out, headers) - end + output_file = Tempfile.new(binmode: true) + http_get_streaming(uncached_url(url, headers), output_file, headers) @downloaded_source = url @source = after_download(url) + + output_file end def before_download(url, datums) @@ -91,7 +89,8 @@ module Downloads end if res.success? - return + file.rewind + return file else raise Error.new("HTTP error code: #{res.code} #{res.message}") end diff --git a/app/logical/null_backup_service.rb b/app/logical/null_backup_service.rb deleted file mode 100644 index 3c15b0f10..000000000 --- a/app/logical/null_backup_service.rb +++ /dev/null @@ -1,9 +0,0 @@ -class NullBackupService - def backup(file_path, options = {}) - # do nothing - end - - def delete(file_path, options = {}) - # do nothing - end -end diff --git a/app/logical/pixiv_ugoira_converter.rb b/app/logical/pixiv_ugoira_converter.rb index abe619656..791e14642 100644 --- a/app/logical/pixiv_ugoira_converter.rb +++ b/app/logical/pixiv_ugoira_converter.rb @@ -1,13 +1,9 @@ class PixivUgoiraConverter - def self.convert(source_path, output_path, preview_path, frame_data) - folder = Zip::File.new(source_path) - write_webm(folder, output_path, frame_data) - write_preview(folder, preview_path) - RemoteFileManager.new(output_path).distribute - RemoteFileManager.new(preview_path).distribute - end + def self.generate_webm(ugoira_file, frame_data) + folder = Zip::File.new(ugoira_file.path) + output_file = Tempfile.new(binmode: true) + write_path = output_file.path - def self.write_webm(folder, write_path, frame_data) Dir.mktmpdir do |tmpdir| FileUtils.mkdir_p("#{tmpdir}/images") folder.each_with_index do |file, i| @@ -64,14 +60,17 @@ class PixivUgoiraConverter return end end + + output_file end - def self.write_preview(folder, path) - Dir.mktmpdir do |tmpdir| - file = folder.first - temp_path = File.join(tmpdir, file.name) - file.extract(temp_path) - DanbooruImageResizer.resize(temp_path, path, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - end + def self.generate_preview(ugoira_file) + file = Tempfile.new(binmode: true) + zipfile = Zip::File.new(ugoira_file.path) + zipfile.entries.first.extract(file.path) { true } # 'true' means overwrite the existing tempfile. + + DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) + ensure + file.close! end end diff --git a/app/logical/pixiv_ugoira_service.rb b/app/logical/pixiv_ugoira_service.rb index 248428a8d..58536fa0d 100644 --- a/app/logical/pixiv_ugoira_service.rb +++ b/app/logical/pixiv_ugoira_service.rb @@ -1,32 +1,10 @@ class PixivUgoiraService attr_reader :width, :height, :frame_data, :content_type - def self.regen(post) - service = new() - service.load( - :is_ugoira => true, - :ugoira_frame_data => post.pixiv_ugoira_frame_data.data - ) - service.generate_resizes(post.file_path, post.large_file_path, post.preview_file_path, false) - end - def save_frame_data(post) PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id) end - def generate_resizes(source_path, output_path, preview_path, delay = true) - # Run this a bit in the future to give the upload process time to move the file - if delay - PixivUgoiraConverter.delay(:queue => Socket.gethostname, :run_at => 10.seconds.from_now, :priority => -1).convert(source_path, output_path, preview_path, @frame_data) - else - PixivUgoiraConverter.convert(source_path, output_path, preview_path, @frame_data) - end - - # since the resizes will be delayed, just touch the output file so the - # file distribution wont break - FileUtils.touch([output_path, preview_path]) - end - def calculate_dimensions(source_path) folder = Zip::File.new(source_path) tempfile = Tempfile.new("ugoira-dimensions") diff --git a/app/logical/s3_backup_service.rb b/app/logical/s3_backup_service.rb deleted file mode 100644 index 655ea9914..000000000 --- a/app/logical/s3_backup_service.rb +++ /dev/null @@ -1,52 +0,0 @@ -class S3BackupService < BackupService - attr_reader :client, :bucket - - def initialize(client: nil, bucket: Danbooru.config.aws_s3_bucket_name) - @credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key) - @client = client || Aws::S3::Client.new(credentials: @credentials, region: "us-east-1", logger: Logger.new(STDOUT)) - @bucket = bucket - end - - def backup(file_path, type: nil, **options) - keys = s3_keys(file_path, type) - keys.each do |key| - upload_to_s3(key, file_path) - end - end - - def delete(file_path, type: nil) - keys = s3_keys(file_path, type) - keys.each do |key| - delete_from_s3(key) - end - end - -protected - def s3_keys(file_path, type) - name = File.basename(file_path) - - case type - when :original - [name] - when :preview - ["preview/#{name}"] - when :large - ["sample/#{name}"] - else - raise ArgumentError.new("Unknown type: #{type}") - end - end - - def delete_from_s3(key) - client.delete_object(bucket: bucket, key: key) - rescue Aws::S3::Errors::NoSuchKey - # ignore - end - - def upload_to_s3(key, file_path) - File.open(file_path, "rb") do |body| - base64_md5 = Digest::MD5.base64digest(File.read(file_path)) - client.put_object(acl: "public-read", bucket: bucket, key: key, body: body, content_md5: base64_md5) - end - end -end diff --git a/app/logical/storage_manager.rb b/app/logical/storage_manager.rb new file mode 100644 index 000000000..1ce833f88 --- /dev/null +++ b/app/logical/storage_manager.rb @@ -0,0 +1,106 @@ +class StorageManager + class Error < StandardError; end + + DEFAULT_BASE_URL = Rails.application.routes.url_helpers.root_url + "data" + DEFAULT_BASE_DIR = "#{Rails.root}/public/data" + + attr_reader :base_url, :base_dir, :hierarchical, :tagged_filenames, :large_image_prefix + + def initialize(base_url: DEFAULT_BASE_URL, base_dir: DEFAULT_BASE_DIR, hierarchical: false, tagged_filenames: Danbooru.config.enable_seo_post_urls, large_image_prefix: Danbooru.config.large_image_prefix) + @base_url = base_url.chomp("/") + @base_dir = base_dir + @hierarchical = hierarchical + @tagged_filenames = tagged_filenames + @large_image_prefix = large_image_prefix + end + + # Store the given file at the given path. If a file already exists at that + # location it should be overwritten atomically. Either the file is fully + # written, or an error is raised and the original file is left unchanged. The + # file should never be in a partially written state. + def store(io, path) + raise NotImplementedError, "store not implemented" + end + + # Delete the file at the given path. If the file doesn't exist, no error + # should be raised. + def delete(path) + raise NotImplementedError, "delete not implemented" + end + + # Return a readonly copy of the file located at the given path. + def open(path) + raise NotImplementedError, "open not implemented" + end + + def store_file(io, post, type) + store(io, file_path(post.md5, post.file_ext, type)) + end + + def delete_file(post_id, md5, file_ext, type) + delete(file_path(md5, file_ext, type)) + end + + def open_file(post, type) + open(file_path(post.md5, post.file_ext, type)) + end + + def file_url(post, type) + subdir = subdir_for(post.md5) + file = file_name(post.md5, post.file_ext, type) + + if type == :preview && !post.has_preview? + "#{base_url}/images/download-preview.png" + elsif type == :preview + "#{base_url}/preview/#{subdir}#{file}" + elsif type == :large && post.has_large? + "#{base_url}/sample/#{subdir}#{seo_tags(post)}#{file}" + else + "#{base_url}/#{subdir}#{seo_tags(post)}#{file}" + end + end + + protected + + def file_path(md5, file_ext, type) + subdir = subdir_for(md5) + file = file_name(md5, file_ext, type) + + case type + when :preview + "#{base_dir}/preview/#{subdir}#{file}" + when :large + "#{base_dir}/sample/#{subdir}#{file}" + when :original + "#{base_dir}/#{subdir}#{file}" + end + end + + def file_name(md5, file_ext, type) + large_file_ext = (file_ext == "zip") ? "webm" : "jpg" + + case type + when :preview + "#{md5}.jpg" + when :large + "#{large_image_prefix}#{md5}.#{large_file_ext}" + when :original + "#{md5}.#{file_ext}" + end + end + + def subdir_for(md5) + if hierarchical + "#{md5[0..1]}/#{md5[2..3]}/" + else + "" + end + end + + def seo_tags(post, user = CurrentUser.user) + return "" if !tagged_filenames || user.disable_tagged_filenames? + + tags = post.humanized_essential_tag_string.gsub(/[^a-z0-9]+/, "_").gsub(/(?:^_+)|(?:_+$)/, "").gsub(/_{2,}/, "_") + "__#{tags}__" + end +end diff --git a/app/logical/storage_manager/hybrid.rb b/app/logical/storage_manager/hybrid.rb new file mode 100644 index 000000000..1e3afdda4 --- /dev/null +++ b/app/logical/storage_manager/hybrid.rb @@ -0,0 +1,23 @@ +class StorageManager::Hybrid < StorageManager + attr_reader :submanager + + def initialize(&block) + @submanager = block + end + + def store_file(io, post, type) + submanager[post.id, post.md5, post.file_ext, type].store_file(io, post, type) + end + + def delete_file(post_id, md5, file_ext, type) + submanager[post_id, md5, file_ext, type].delete_file(post_id, md5, file_ext, type) + end + + def open_file(io, post, type) + submanager[post.id, post.md5, post.file_ext, type].open_file(post, type) + end + + def file_url(post, type) + submanager[post.id, post.md5, post.file_ext, type].file_url(post, type) + end +end diff --git a/app/logical/storage_manager/local.rb b/app/logical/storage_manager/local.rb new file mode 100644 index 000000000..6bdfa69e3 --- /dev/null +++ b/app/logical/storage_manager/local.rb @@ -0,0 +1,25 @@ +class StorageManager::Local < StorageManager + DEFAULT_PERMISSIONS = 0644 + + def store(io, dest_path) + temp_path = dest_path + "-" + SecureRandom.uuid + ".tmp" + + FileUtils.mkdir_p(File.dirname(temp_path)) + bytes_copied = IO.copy_stream(io, temp_path) + raise Error, "store failed: #{bytes_copied}/#{io.size} bytes copied" if bytes_copied != io.size + + FileUtils.chmod(DEFAULT_PERMISSIONS, temp_path) + File.rename(temp_path, dest_path) + rescue StandardError => e + FileUtils.rm_f(temp_path) + raise Error, e + end + + def delete(path) + FileUtils.rm_f(path) + end + + def open(path) + File.open(path, "r", binmode: true) + end +end diff --git a/app/logical/storage_manager/null.rb b/app/logical/storage_manager/null.rb new file mode 100644 index 000000000..ebaeb8bfe --- /dev/null +++ b/app/logical/storage_manager/null.rb @@ -0,0 +1,13 @@ +class StorageManager::Null < StorageManager + def store(io, path) + # no-op + end + + def delete(path) + # no-op + end + + def open(path) + # no-op + end +end diff --git a/app/logical/storage_manager/s3.rb b/app/logical/storage_manager/s3.rb new file mode 100644 index 000000000..6549bf0bf --- /dev/null +++ b/app/logical/storage_manager/s3.rb @@ -0,0 +1,43 @@ +class StorageManager::S3 < StorageManager + # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method + DEFAULT_S3_OPTIONS = { + region: Danbooru.config.aws_region, + credentials: Danbooru.config.aws_credentials, + logger: Rails.logger, + } + + # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#put_object-instance_method + DEFAULT_PUT_OPTIONS = { + acl: "public-read", + storage_class: "STANDARD", # STANDARD, STANDARD_IA, REDUCED_REDUNDANCY + cache_control: "public, max-age=#{1.year.to_i}", + #content_type: "image/jpeg" # XXX should set content type + } + + attr_reader :bucket, :client, :s3_options + + def initialize(bucket, client: nil, s3_options: {}, **options) + @bucket = bucket + @s3_options = DEFAULT_S3_OPTIONS.merge(s3_options) + @client = client || Aws::S3::Client.new(**@s3_options) + super(**options) + end + + def store(io, path) + data = io.read + base64_md5 = Digest::MD5.base64digest(data) + client.put_object(bucket: bucket, key: path, body: data, content_md5: base64_md5, **DEFAULT_PUT_OPTIONS) + end + + def delete(path) + client.delete_object(bucket: bucket, key: path) + rescue Aws::S3::Errors::NoSuchKey + # ignore + end + + def open(path) + file = Tempfile.new(binmode: true) + client.get_object(bucket: bucket: key: path, response_target: file) + file + end +end diff --git a/app/logical/storage_manager/sftp.rb b/app/logical/storage_manager/sftp.rb new file mode 100644 index 000000000..add10d010 --- /dev/null +++ b/app/logical/storage_manager/sftp.rb @@ -0,0 +1,76 @@ +class StorageManager::SFTP < StorageManager + DEFAULT_PERMISSIONS = 0644 + + # http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start + DEFAULT_SSH_OPTIONS = { + timeout: 10, + logger: Rails.logger, + verbose: :fatal, + non_interactive: true, + } + + attr_reader :hosts, :ssh_options + + def initialize(*hosts, ssh_options: {}, **options) + @hosts = hosts + @ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options) + super(**options) + end + + def store(file, dest_path) + temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp" + dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak" + + each_host do |host, sftp| + begin + sftp.upload!(file.path, temp_upload_path) + sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS) + + # `rename!` can't overwrite existing files, so if a file already exists + # at dest_path we move it out of the way first. + force { sftp.rename!(dest_path, dest_backup_path) } + force { sftp.rename!(temp_upload_path, dest_path) } + rescue StandardError => e + # if anything fails, try to move the original file back in place (if it was moved). + force { sftp.rename!(dest_backup_path, dest_path) } + raise Error, e + ensure + force { sftp.remove!(temp_upload_path) } + force { sftp.remove!(dest_backup_path) } + end + end + end + + def delete(dest_path) + each_host do |host, sftp| + force { sftp.remove!(dest_path) } + end + end + + def open(dest_path) + file = Tempfile.new(binmode: true) + + Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp| + sftp.download!(dest_path, file.path) + end + + file + end + + protected + + # Ignore "no such file" exceptions for the given operation. + def force + yield + rescue Net::SFTP::StatusException => e + raise Error, e unless e.description == "no such file" + end + + def each_host + hosts.each do |host| + Net::SFTP.start(host, nil, ssh_options) do |sftp| + yield host, sftp + end + end + end +end diff --git a/app/models/post.rb b/app/models/post.rb index 4aa4e9649..9e6a211e0 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -29,7 +29,6 @@ class Post < ApplicationRecord before_save :set_tag_counts before_save :set_pool_category_pseudo_tags before_create :autoban - after_save :queue_backup, if: :md5_changed? after_save :create_version after_save :update_parent_on_save after_save :apply_post_metatags @@ -94,144 +93,76 @@ class Post < ApplicationRecord extend ActiveSupport::Concern module ClassMethods - def delete_files(post_id, file_path, large_file_path, preview_file_path, force: false) - unless force - # XXX should pass in the md5 instead of parsing it. - preview_file_path =~ %r!/data/preview/(?:test\.)?([a-z0-9]{32})\.jpg\z! - md5 = $1 - - if Post.where(md5: md5).exists? - raise DeletionError.new("Files still in use; skipping deletion.") - end + def delete_files(post_id, md5, file_ext, force: false) + if Post.where(md5: md5).exists? && !force + raise DeletionError.new("Files still in use; skipping deletion.") end - backup_service = Danbooru.config.backup_service - backup_service.delete(file_path, type: :original) - backup_service.delete(large_file_path, type: :large) - backup_service.delete(preview_file_path, type: :preview) + Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :original) + Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :large) + Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :preview) - # the large file and the preview don't necessarily exist. if so errors will be ignored. - FileUtils.rm_f(file_path) - FileUtils.rm_f(large_file_path) - FileUtils.rm_f(preview_file_path) - - RemoteFileManager.new(file_path).delete - RemoteFileManager.new(large_file_path).delete - RemoteFileManager.new(preview_file_path).delete + Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :original) + Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :large) + Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :preview) if Danbooru.config.cloudflare_key - md5, ext = File.basename(file_path).split(".") - CloudflareService.new.delete(md5, ext) + CloudflareService.new.delete(md5, file_ext) end end end + def queue_delete_files(grace_period) + Post.delay(queue: "default", run_at: Time.now + grace_period).delete_files(id, md5, file_ext) + end + def delete_files - Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true) + Post.delete_files(id, md5, file_ext, force: true) end - def distribute_files - if Danbooru.config.build_file_url(self) =~ /^http/ - # this post is archived - RemoteFileManager.new(file_path).distribute_to_archive(Danbooru.config.build_file_url(self)) - RemoteFileManager.new(preview_file_path).distribute if has_preview? - RemoteFileManager.new(large_file_path).distribute_to_archive(Danbooru.config.build_large_file_url(self)) if has_large? - else - RemoteFileManager.new(file_path).distribute - RemoteFileManager.new(preview_file_path).distribute if has_preview? - RemoteFileManager.new(large_file_path).distribute if has_large? - end + def distribute_files(file, sample_file, preview_file) + storage_manager.store_file(file, self, :original) + storage_manager.store_file(sample_file, self, :large) if sample_file.present? + storage_manager.store_file(preview_file, self, :preview) if preview_file.present? + + backup_storage_manager.store_file(file, self, :original) + backup_storage_manager.store_file(sample_file, self, :large) if sample_file.present? + backup_storage_manager.store_file(preview_file, self, :preview) if preview_file.present? end - def file_path_prefix - Rails.env == "test" ? "test." : "" + def backup_storage_manager + Danbooru.config.backup_storage_manager end - def file_path - "#{Rails.root}/public/data/#{file_path_prefix}#{md5}.#{file_ext}" + def storage_manager + Danbooru.config.storage_manager end - def large_file_path - if has_large? - "#{Rails.root}/public/data/sample/#{file_path_prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}" - else - file_path - end - end - - def large_file_ext - if is_ugoira? - "webm" - else - "jpg" - end - end - - def preview_file_path - "#{Rails.root}/public/data/preview/#{file_path_prefix}#{md5}.jpg" - end - - def file_name - "#{file_path_prefix}#{md5}.#{file_ext}" + def file(type = :original) + storage_manager.open_file(self, type) end def file_url - Danbooru.config.build_file_url(self) - end - - # this is for the 640x320 version - def cropped_file_url + storage_manager.file_url(self, :original) end def large_file_url - if has_large? - Danbooru.config.build_large_file_url(self) - else - file_url - end - end - - def seo_tag_string - if Danbooru.config.enable_seo_post_urls && !CurrentUser.user.disable_tagged_filenames? - "__#{seo_tags}__" - else - nil - end - end - - def seo_tags - @seo_tags ||= humanized_essential_tag_string.gsub(/[^a-z0-9]+/, "_").gsub(/(?:^_+)|(?:_+$)/, "").gsub(/_{2,}/, "_") + storage_manager.file_url(self, :large) end def preview_file_url - if !has_preview? - return "/images/download-preview.png" - end - - "/data/preview/#{file_path_prefix}#{md5}.jpg" - end - - def complete_preview_file_url - "http://#{Danbooru.config.hostname}#{preview_file_url}" + storage_manager.file_url(self, :preview) end def open_graph_image_url if is_image? if has_large? - if Danbooru.config.build_large_file_url(self) =~ /http/ - large_file_url - else - "http://#{Danbooru.config.hostname}#{large_file_url}" - end + large_file_url else - if Danbooru.config.build_file_url(self) =~ /http/ - file_url - else - "http://#{Danbooru.config.hostname}#{file_url}" - end + file_url end else - complete_preview_file_url + preview_file_url end end @@ -243,36 +174,10 @@ class Post < ApplicationRecord end end - def file_path_for(user) - if user.default_image_size == "large" && image_width > Danbooru.config.large_image_width - large_file_path - else - file_path - end - end - def is_image? file_ext =~ /jpg|jpeg|gif|png/i end - def is_animated_gif? - if file_ext =~ /gif/i && File.exists?(file_path) - return Magick::Image.ping(file_path).length > 1 - else - return false - end - end - - def is_animated_png? - if file_ext =~ /png/i && File.exists?(file_path) - apng = APNGInspector.new(file_path) - apng.inspect! - return apng.animated? - else - return false - end - end - def is_flash? file_ext =~ /swf/i end @@ -294,9 +199,7 @@ class Post < ApplicationRecord end def has_preview? - # for video/ugoira we don't want to try and render a preview that - # might doesn't exist yet - is_image? || ((is_video? || is_ugoira?) && File.exists?(preview_file_path)) + is_image? || is_video? || is_ugoira? end def has_dimensions? @@ -304,24 +207,7 @@ class Post < ApplicationRecord end def has_ugoira_webm? - created_at < 1.minute.ago || (File.exists?(preview_file_path) && File.size(preview_file_path) > 0) - end - end - - module BackupMethods - extend ActiveSupport::Concern - - def queue_backup - Post.delay(queue: "default", priority: -1).backup_file(file_path, id: id, type: :original) - Post.delay(queue: "default", priority: -1).backup_file(large_file_path, id: id, type: :large) if has_large? - Post.delay(queue: "default", priority: -1).backup_file(preview_file_path, id: id, type: :preview) if has_preview? - end - - module ClassMethods - def backup_file(file_path, options = {}) - backup_service = Danbooru.config.backup_service - backup_service.backup(file_path, options) - end + true end end @@ -765,7 +651,6 @@ class Post < ApplicationRecord return tags if !Danbooru.config.enable_dimension_autotagging tags -= %w(incredibly_absurdres absurdres highres lowres huge_filesize flash webm mp4) - tags -= %w(animated_gif animated_png) if new_record? if has_dimensions? if image_width >= 10_000 || image_height >= 10_000 @@ -794,14 +679,6 @@ class Post < ApplicationRecord tags << "huge_filesize" end - if is_animated_gif? - tags << "animated_gif" - end - - if is_animated_png? - tags << "animated_png" - end - if is_flash? tags << "flash" end @@ -1747,8 +1624,8 @@ class Post < ApplicationRecord end def update_iqdb_async - if File.exists?(preview_file_path) && Post.iqdb_enabled? - Post.iqdb_sqs_service.send_message("update\n#{id}\n#{complete_preview_file_url}") + if Post.iqdb_enabled? + Post.iqdb_sqs_service.send_message("update\n#{id}\n#{preview_file_url}") end end @@ -1854,7 +1731,6 @@ class Post < ApplicationRecord end include FileMethods - include BackupMethods include ImageMethods include ApprovalMethods include PresenterMethods diff --git a/app/models/post_replacement.rb b/app/models/post_replacement.rb index aefe2715c..e709942fb 100644 --- a/app/models/post_replacement.rb +++ b/app/models/post_replacement.rb @@ -24,6 +24,8 @@ class PostReplacement < ApplicationRecord end def process! + upload = nil + transaction do upload = Upload.create!( file: replacement_file, @@ -47,7 +49,7 @@ class PostReplacement < ApplicationRecord # md5/file_ext to delete the old files. if saving the post fails, # this is rolled back so the job won't run. if md5_changed - Post.delay(queue: "default", run_at: Time.now + DELETION_GRACE_PERIOD).delete_files(post.id, post.file_path, post.large_file_path, post.preview_file_path) + post.queue_delete_files(DELETION_GRACE_PERIOD) end self.file_ext = upload.file_ext @@ -69,8 +71,6 @@ class PostReplacement < ApplicationRecord if md5_changed post.comments.create!({creator: User.system, body: comment_replacement_message, do_not_bump_post: true}, without_protection: true) - else - post.queue_backup end save! @@ -79,7 +79,7 @@ class PostReplacement < ApplicationRecord # point of no return: these things can't be rolled back, so we do them # only after the transaction successfully commits. - post.distribute_files + upload.distribute_files(post) post.update_iqdb_async end diff --git a/app/models/upload.rb b/app/models/upload.rb index 8bf5a7872..6b241c4d4 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -10,14 +10,11 @@ class Upload < ApplicationRecord belongs_to :uploader, :class_name => "User" belongs_to :post before_validation :initialize_uploader, :on => :create - before_validation :initialize_status, :on => :create - before_create :convert_cgi_file - after_destroy :delete_temp_file validate :uploader_is_not_limited, :on => :create validate :file_or_source_is_present, :on => :create validate :rating_given attr_accessible :file, :image_width, :image_height, :file_ext, :md5, - :file_size, :as_pending, :source, :file_path, :content_type, :rating, + :file_size, :as_pending, :source, :rating, :tag_string, :status, :backtrace, :post_id, :md5_confirmation, :parent_id, :server, :artist_commentary_title, :artist_commentary_desc, :include_artist_commentary, @@ -53,12 +50,6 @@ class Upload < ApplicationRecord end end - def validate_file_exists - unless file_path && File.exists?(file_path) - raise "file does not exist" - end - end - def validate_file_content_type unless is_valid_content_type? raise "invalid content type (only JPEG, PNG, GIF, SWF, MP4, and WebM files are allowed)" @@ -75,12 +66,6 @@ class Upload < ApplicationRecord end end - def validate_md5_confirmation_after_move - if !md5_confirmation.blank? && md5_confirmation != Digest::MD5.file(md5_file_path).hexdigest - raise "md5 mismatch" - end - end - def rating_given if rating.present? return true @@ -93,10 +78,14 @@ class Upload < ApplicationRecord end end - def tag_audio - if is_video? && video.audio_channels.present? - self.tag_string = "#{tag_string} video_with_sound" - end + def automatic_tags + return "" unless Danbooru.config.enable_dimension_autotagging + + tags = [] + tags << "video_with_sound" if is_video_with_audio? + tags << "animated_gif" if is_animated_gif? + tags << "animated_png" if is_animated_png? + tags.join(" ") end def validate_video_duration @@ -110,35 +99,36 @@ class Upload < ApplicationRecord module ConversionMethods def process_upload - CurrentUser.scoped(uploader, uploader_ip_addr) do + begin update_attribute(:status, "processing") - self.source = strip_source + + self.source = source.to_s.strip if is_downloadable? - self.downloaded_source, self.source = download_from_source(temp_file_path) + self.downloaded_source, self.source, self.file = download_from_source(source, referer_url) + else + self.file = self.file.tempfile end - validate_file_exists - self.content_type = file_header_to_content_type(file_path) - self.file_ext = content_type_to_file_ext(content_type) + + self.file_ext = file_header_to_file_ext(file) + self.file_size = file.size + self.md5 = Digest::MD5.file(file.path).hexdigest + validate_file_content_type - calculate_hash(file_path) validate_md5_uniqueness validate_md5_confirmation - tag_audio validate_video_duration - calculate_file_size(file_path) - if has_dimensions? - calculate_dimensions(file_path) - end - generate_resizes(file_path) - move_file - validate_md5_confirmation_after_move + + self.tag_string = "#{tag_string} #{automatic_tags}" + self.image_width, self.image_height = calculate_dimensions + save end end def create_post_from_upload post = convert_to_post - post.distribute_files + distribute_files(post) + if post.save create_artist_commentary(post) if include_artist_commentary? ugoira_service.save_frame_data(post) if is_ugoira? @@ -151,6 +141,14 @@ class Upload < ApplicationRecord post end + def distribute_files(post) + preview_file, sample_file = generate_resizes + post.distribute_files(file, sample_file, preview_file) + ensure + preview_file.try(:close!) + sample_file.try(:close!) + end + def process!(force = false) @tries ||= 0 return if !force && status =~ /processing|completed|error/ @@ -170,13 +168,9 @@ class Upload < ApplicationRecord rescue Exception => x update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n")) nil - - ensure - delete_temp_file - end - def async_conversion? - is_ugoira? + ensure + file.try(:close!) end def ugoira_service @@ -211,23 +205,6 @@ class Upload < ApplicationRecord end module FileMethods - def delete_temp_file(path = nil) - FileUtils.rm_f(path || temp_file_path) - end - - def move_file - FileUtils.mv(file_path, md5_file_path) - end - - def calculate_file_size(source_path) - self.file_size = File.size(source_path) - end - - # Calculates the MD5 based on whatever is in temp_file_path - def calculate_hash(source_path) - self.md5 = Digest::MD5.file(source_path).hexdigest - end - def is_image? %w(jpg gif png).include?(file_ext) end @@ -240,75 +217,68 @@ class Upload < ApplicationRecord %w(webm mp4).include?(file_ext) end + def is_video_with_audio? + is_video? && video.audio_channels.present? + end + def is_ugoira? %w(zip).include?(file_ext) end + + def is_animated_gif? + file_ext == "gif" && Magick::Image.ping(file.path).length > 1 + end + + def is_animated_png? + file_ext == "png" && APNGInspector.new(file.path).inspect!.animated? + end end module ResizerMethods - def generate_resizes(source_path) - generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85) + def generate_resizes + if is_video? + preview_file = generate_video_preview_for(video, width, height) + elsif is_ugoira? + preview_file = PixivUgoiraConverter.generate_preview(file) + sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data) + elsif is_image? + preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - if is_image? && image_width > Danbooru.config.large_image_width - generate_resize_for(Danbooru.config.large_image_width, nil, source_path) + if image_width > Danbooru.config.large_image_width + sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, nil, 90) + end end + + [preview_file, sample_file] end - def generate_video_preview_for(width, height, output_path) - dimension_ratio = image_width.to_f / image_height + def generate_video_preview_for(video, width, height) + dimension_ratio = video.width.to_f / video.height if dimension_ratio > 1 height = (width / dimension_ratio).to_i else width = (height * dimension_ratio).to_i end - video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) - FileUtils.chmod(0664, output_path) - end - def generate_resize_for(width, height, source_path, quality = 90) - unless File.exists?(source_path) - raise Error.new("file not found") - end - - output_path = resized_file_path_for(width) - if is_image? - DanbooruImageResizer.resize(source_path, output_path, width, height, quality) - elsif is_ugoira? - if Delayed::Worker.delay_jobs - # by the time this runs we'll have moved source_path to md5_file_path - ugoira_service.generate_resizes(md5_file_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width)) - else - ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width), false) - end - elsif is_video? - generate_video_preview_for(width, height, output_path) - end + output_file = Tempfile.new(binmode: true) + video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) + output_file end end module DimensionMethods # Figures out the dimensions of the image. - def calculate_dimensions(file_path) + def calculate_dimensions if is_video? - self.image_width = video.width - self.image_height = video.height + [video.width, video.height] elsif is_ugoira? - ugoira_service.calculate_dimensions(file_path) - self.image_width = ugoira_service.width - self.image_height = ugoira_service.height + ugoira_service.calculate_dimensions(file.path) + [ugoira_service.width, ugoira_service.height] else - File.open(file_path, "rb") do |file| - image_size = ImageSpec.new(file) - self.image_width = image_size.width - self.image_height = image_size.height - end + image_size = ImageSpec.new(file.path) + [image_size.width, image_size.height] end end - - # Does this file have image dimensions? - def has_dimensions? - %w(jpg gif png swf webm mp4 zip).include?(file_ext) - end end module ContentTypeMethods @@ -316,136 +286,44 @@ class Upload < ApplicationRecord file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/ end - def content_type_to_file_ext(content_type) - case content_type - when "image/jpeg" + def file_header_to_file_ext(file) + case File.read(file.path, 16) + when /^\xff\xd8/n "jpg" - - when "image/gif" + when /^GIF87a/, /^GIF89a/ "gif" - - when "image/png" + when /^\x89PNG\r\n\x1a\n/n "png" - - when "application/x-shockwave-flash" + when /^CWS/, /^FWS/, /^ZWS/ "swf" - - when "video/webm" + when /^\x1a\x45\xdf\xa3/n "webm" - - when "video/mp4" + when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/ "mp4" - - when "application/zip" + when /^PK\x03\x04/ "zip" - else "bin" end end - - def file_header_to_content_type(source_path) - case File.read(source_path, 16) - when /^\xff\xd8/n - "image/jpeg" - - when /^GIF87a/, /^GIF89a/ - "image/gif" - - when /^\x89PNG\r\n\x1a\n/n - "image/png" - - when /^CWS/, /^FWS/, /^ZWS/ - "application/x-shockwave-flash" - - when /^\x1a\x45\xdf\xa3/n - "video/webm" - - when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/ - "video/mp4" - - when /^PK\x03\x04/ - "application/zip" - - else - "application/octet-stream" - end - end - end - - module FilePathMethods - def md5_file_path - prefix = Rails.env == "test" ? "test." : "" - "#{Rails.root}/public/data/#{prefix}#{md5}.#{file_ext}" - end - - def resized_file_path_for(width) - prefix = Rails.env == "test" ? "test." : "" - - case width - when Danbooru.config.small_image_width - "#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg" - - when Danbooru.config.large_image_width - "#{Rails.root}/public/data/sample/#{prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}" - end - end - - def large_file_ext - if is_ugoira? - "webm" - else - "jpg" - end - end - - def temp_file_path - @temp_file_path ||= File.join(Rails.root, "tmp", "upload_#{Time.now.to_f}.#{Process.pid}") - end end module DownloaderMethods - def strip_source - source.to_s.strip - end - # Determines whether the source is downloadable def is_downloadable? - source =~ /^https?:\/\// && file_path.blank? + source =~ /^https?:\/\// && file.blank? end - # Downloads the file to destination_path - def download_from_source(destination_path) - self.file_path = destination_path - download = Downloads::File.new(source, destination_path, :referer_url => referer_url) - download.download! + def download_from_source(source, referer_url) + download = Downloads::File.new(source, referer_url: referer_url) + file = download.download! ugoira_service.load(download.data) - [download.downloaded_source, download.source] - end - end - module CgiFileMethods - def convert_cgi_file - return if file.blank? || file.size == 0 - - self.file_path = temp_file_path - - if file.respond_to?(:tempfile) && file.tempfile - FileUtils.cp(file.tempfile.path, file_path) - else - File.open(file_path, 'wb') do |out| - out.write(file.read) - end - end - FileUtils.chmod(0664, file_path) + [download.downloaded_source, download.source, file] end end module StatusMethods - def initialize_status - self.status = "pending" - end - def is_pending? status == "pending" end @@ -480,7 +358,7 @@ class Upload < ApplicationRecord module VideoMethods def video - @video ||= FFMPEG::Movie.new(file_path) + @video ||= FFMPEG::Movie.new(file.path) end end @@ -534,8 +412,6 @@ class Upload < ApplicationRecord include DimensionMethods include ContentTypeMethods include DownloaderMethods - include FilePathMethods - include CgiFileMethods include StatusMethods include UploaderMethods include VideoMethods diff --git a/app/presenters/post_presenter.rb b/app/presenters/post_presenter.rb index 4fe901fbb..e0c70520a 100644 --- a/app/presenters/post_presenter.rb +++ b/app/presenters/post_presenter.rb @@ -135,6 +135,10 @@ class PostPresenter < Presenter @post.humanized_essential_tag_string end + def filename_for_download + "#{humanized_essential_tag_string} - #{@post.md5}.#{@post.file_ext}" + end + def categorized_tag_groups string = [] diff --git a/app/views/comments/index.atom.builder b/app/views/comments/index.atom.builder index 09d017868..19e1c8b75 100644 --- a/app/views/comments/index.atom.builder +++ b/app/views/comments/index.atom.builder @@ -11,7 +11,7 @@ atom_feed(root_url: comments_url(host: Danbooru.config.hostname)) do |feed| feed.entry(comment, published: comment.created_at, updated: comment.updated_at) do |entry| entry.title("@#{comment.creator_name} on post ##{comment.post_id} (#{comment.post.humanized_essential_tag_string})") entry.content(<<-EOS.strip_heredoc, type: "html") - + #{format_text(comment.body)} EOS diff --git a/app/views/posts/partials/show/_options.html.erb b/app/views/posts/partials/show/_options.html.erb index 81350deaa..79d83588d 100644 --- a/app/views/posts/partials/show/_options.html.erb +++ b/app/views/posts/partials/show/_options.html.erb @@ -4,7 +4,7 @@ <% if CurrentUser.is_member? %>
  • <%= link_to "Favorite", favorites_path(:post_id => post.id), :remote => true, :method => :post, :id => "add-to-favorites", :title => "Shortcut is F" %>
  • <%= link_to "Unfavorite", favorite_path(post), :remote => true, :method => :delete, :id => "remove-from-favorites" %>
    • -
  • <%= link_to_if post.visible?, "Download", post.file_url, :download => post.presenter.humanized_essential_tag_string + " - " + post.file_name %>
    • +
  • <%= link_to_if post.visible?, "Download", post.file_url, download: post.presenter.filename_for_download %>
  • <%= link_to "Add to pool", "#", :id => "pool" %>
    • <% if post.is_note_locked? %>
  • Note locked
    • diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index fd30ddc0c..ecbb9b63f 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -99,20 +99,6 @@ module Danbooru true end - # What method to use to backup images. - # - # NullBackupService: Don't backup images at all. - # - # S3BackupService: Backup to Amazon S3. Must configure aws_access_key_id, - # aws_secret_access_key, and aws_s3_bucket_name. Bucket must exist and be writable. - def backup_service - if Rails.env.production? - S3BackupService.new - else - NullBackupService.new - end - end - # What method to use to store images. # local_flat: Store every image in one directory. # local_hierarchy: Store every image in a hierarchical directory, based on the post's MD5 hash. On some file systems this may be faster. @@ -222,12 +208,58 @@ module Danbooru "danbooru" end - def build_file_url(post) - "/data/#{post.file_path_prefix}/#{post.md5}.#{post.file_ext}" + # The method to use for storing image files. + def storage_manager + # Store files on the local filesystem. + # base_dir - where to store files (default: under public/data) + # base_url - where to serve files from (default: http://#{hostname}/data) + # hierarchical: false - store files in a single directory + # hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash + StorageManager::Local.new(base_dir: "#{Rails.root}/public/data", hierarchical: false) + + # Store files on one or more remote host(s). Configure SSH settings in + # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) + # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {}) + + # Store files in an S3 bucket. The bucket must already exist and be + # writable by you. Configure your S3 settings in aws_region and + # aws_credentials below, or in the s3_options param (ref: + # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method) + # StorageManager::S3.new("my_s3_bucket", base_url: "https://my_s3_bucket.s3.amazonaws.com/", s3_options: {}) + + # Select the storage method based on the post's id and type (preview, large, or original). + # StorageManager::Hybrid.new do |id, md5, file_ext, type| + # ssh_options = { user: "danbooru" } + # + # if type.in?([:large, :original]) && id.in?(0..850_000) + # StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options) + # elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000) + # StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options) + # elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000) + # StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options) + # else + # StorageManager::SFTP.new(*all_server_hosts, ssh_options: ssh_options) + # end + # end end - def build_large_file_url(post) - "/data/sample/#{post.file_path_prefix}#{Danbooru.config.large_image_prefix}#{post.md5}.#{post.large_file_ext}" + # The method to use for backing up image files. + def backup_storage_manager + # Don't perform any backups. + StorageManager::Null.new + + # Backup files to /mnt/backup on the local filesystem. + # StorageManager::Local.new(base_dir: "/mnt/backup", hierarchical: false) + + # Backup files to /mnt/backup on a remote system. Configure SSH settings + # in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) + # StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {}) + + # Backup files to an S3 bucket. The bucket must already exist and be + # writable by you. Configure your S3 settings in aws_region and + # aws_credentials below, or in the s3_options param (ref: + # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method) + # StorageManager::S3.new("my_s3_bucket_name", s3_options: {}) end #TAG CONFIGURATION @@ -611,6 +643,14 @@ module Danbooru end # AWS config options + def aws_region + "us-east-1" + end + + def aws_credentials + Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key) + end + def aws_access_key_id end diff --git a/test/factories/upload.rb b/test/factories/upload.rb index c4ac00104..cd6a89d0d 100644 --- a/test/factories/upload.rb +++ b/test/factories/upload.rb @@ -1,5 +1,3 @@ -require 'fileutils' - FactoryGirl.define do factory(:upload) do rating "s" @@ -15,51 +13,10 @@ FactoryGirl.define do end factory(:jpg_upload) do - content_type "image/jpeg" file do f = Tempfile.new - f.write(File.read("#{Rails.root}/test/files/test.jpg")) - f.seek(0) - f - end - end - - factory(:exif_jpg_upload) do - content_type "image/jpeg" - file_path do - FileUtils.cp("#{Rails.root}/test/files/test-exif-small.jpg", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test-exif-small.jpg" - end - end - - factory(:blank_jpg_upload) do - content_type "image/jpeg" - file_path do - FileUtils.cp("#{Rails.root}/test/files/test-blank.jpg", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test-blank.jpg" - end - end - - factory(:large_jpg_upload) do - file_ext "jpg" - content_type "image/jpeg" - file_path do - FileUtils.cp("#{Rails.root}/test/files/test-large.jpg", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test-large.jpg" - end - end - - factory(:png_upload) do - file_path do - FileUtils.cp("#{Rails.root}/test/files/test.png", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test.png" - end - end - - factory(:gif_upload) do - file_path do - FileUtils.cp("#{Rails.root}/test/files/test.gif", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test.gif" + IO.copy_stream("#{Rails.root}/test/files/test.jpg", f.path) + ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "test.jpg") end end end diff --git a/test/functional/post_replacements_controller_test.rb b/test/functional/post_replacements_controller_test.rb index c4d05597e..5944215de 100644 --- a/test/functional/post_replacements_controller_test.rb +++ b/test/functional/post_replacements_controller_test.rb @@ -39,7 +39,7 @@ class PostReplacementsControllerTest < ActionController::TestCase assert_response :success assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source) assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5) - assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file(:original)).hexdigest) end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 04a3d3edd..9a7dd29da 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -38,9 +38,14 @@ class ActiveSupport::TestCase mock_missed_search_service! WebMock.allow_net_connect! Danbooru.config.stubs(:enable_sock_puppet_validation?).returns(false) + + storage_manager = StorageManager::Local.new(base_dir: "#{Rails.root}/public/data/test") + Danbooru.config.stubs(:storage_manager).returns(storage_manager) + Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new) end teardown do + FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir) Cache.clear end end diff --git a/test/test_helpers/download_helper.rb b/test/test_helpers/download_helper.rb index 9097e7cea..8381f4629 100644 --- a/test/test_helpers/download_helper.rb +++ b/test/test_helpers/download_helper.rb @@ -1,18 +1,14 @@ module DownloadTestHelper def assert_downloaded(expected_filesize, source) - tempfile = Tempfile.new("danbooru-test") - download = Downloads::File.new(source, tempfile.path) - assert_nothing_raised(Downloads::File::Error) do - download.download! + tempfile = Downloads::File.new(source).download! + assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}") + tempfile.close! end - - assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}") end def assert_rewritten(expected_source, test_source) - tempfile = Tempfile.new("danbooru-test") - download = Downloads::File.new(test_source, tempfile.path) + download = Downloads::File.new(test_source) rewritten_source, _, _ = download.before_download(test_source, {}) assert_match(expected_source, rewritten_source, "Tested source URL: #{test_source}") diff --git a/test/test_helpers/iqdb_test_helper.rb b/test/test_helpers/iqdb_test_helper.rb index 9bcdec4f4..edd941f6e 100644 --- a/test/test_helpers/iqdb_test_helper.rb +++ b/test/test_helpers/iqdb_test_helper.rb @@ -23,7 +23,7 @@ module IqdbTestHelper end def mock_iqdb_matches!(post_or_source, matches) - source = post_or_source.is_a?(Post) ? post_or_source.complete_preview_file_url : post_or_source + source = post_or_source.is_a?(Post) ? post_or_source.preview_file_url : post_or_source url = "http://localhost:3004/similar?key=hunter2&url=#{CGI.escape source}&ref" body = matches.map { |post| { post_id: post.id } }.to_json diff --git a/test/test_helpers/upload_test_helper.rb b/test/test_helpers/upload_test_helper.rb index 3b29574e6..66b404278 100644 --- a/test/test_helpers/upload_test_helper.rb +++ b/test/test_helpers/upload_test_helper.rb @@ -1,23 +1,10 @@ module UploadTestHelper - def upload_file(path, content_type, filename) - tempfile = Tempfile.new(filename) - FileUtils.copy_file(path, tempfile.path) + def upload_file(path) + file = Tempfile.new(binmode: true) + IO.copy_stream("#{Rails.root}/#{path}", file.path) + uploaded_file = ActionDispatch::Http::UploadedFile.new(tempfile: file, filename: File.basename(path)) - (class << tempfile; self; end).class_eval do - alias local_path path - define_method(:tempfile) {self} - define_method(:original_filename) {filename} - define_method(:content_type) {content_type} - end - - tempfile - end - - def upload_jpeg(path) - upload_file(path, "image/jpeg", File.basename(path)) - end - - def upload_zip(path) - upload_file(path, "application/zip", File.basename(path)) + yield uploaded_file if block_given? + uploaded_file end end diff --git a/test/unit/downloads/art_station_test.rb b/test/unit/downloads/art_station_test.rb index 077007b8d..528210bdb 100644 --- a/test/unit/downloads/art_station_test.rb +++ b/test/unit/downloads/art_station_test.rb @@ -5,9 +5,7 @@ module Downloads context "a download for a (small) artstation image" do setup do @source = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - @download.download! + @download = Downloads::File.new(@source) end should "download the large image instead" do @@ -18,8 +16,7 @@ module Downloads context "for an image where an original does not exist" do setup do @source = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) + @download = Downloads::File.new(@source) @download.download! end @@ -38,8 +35,7 @@ module Downloads context "a download for a https://$artist.artstation.com/projects/$id page" do setup do @source = "https://dantewontdie.artstation.com/projects/YZK5q" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) + @download = Downloads::File.new(@source) @download.download! end diff --git a/test/unit/downloads/deviant_art_test.rb b/test/unit/downloads/deviant_art_test.rb index 42013174d..7cb58c5bd 100644 --- a/test/unit/downloads/deviant_art_test.rb +++ b/test/unit/downloads/deviant_art_test.rb @@ -5,9 +5,8 @@ module Downloads context "a download for a deviant art html page" do setup do @source = "http://starbitt.deviantart.com/art/09271X-636962118" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - @download.download! + @download = Downloads::File.new(@source) + @tempfile = @download.download! end should "set the html page as the source" do diff --git a/test/unit/downloads/file_test.rb b/test/unit/downloads/file_test.rb index 126b4f174..7698941d8 100644 --- a/test/unit/downloads/file_test.rb +++ b/test/unit/downloads/file_test.rb @@ -5,12 +5,7 @@ module Downloads context "A twitter video download" do setup do @source = "https://twitter.com/CincinnatiZoo/status/859073537713328129" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - end - - teardown do - @tempfile.close + @download = Downloads::File.new(@source) end should "preserve the twitter source" do @@ -22,12 +17,8 @@ module Downloads context "A post download" do setup do @source = "http://www.google.com/intl/en_ALL/images/logo.gif" + @download = Downloads::File.new(@source) @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - end - - teardown do - @tempfile.close end context "that fails" do @@ -49,10 +40,9 @@ module Downloads end should "store the file in the tempfile path" do - @download.download! + tempfile = @download.download! assert_equal(@source, @download.source) - assert(::File.exists?(@tempfile.path), "temp file should exist") - assert(::File.size(@tempfile.path) > 0, "should have data") + assert_operator(tempfile.size, :>, 0, "should have data") end end end diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb index 66cabf113..3441ee81b 100644 --- a/test/unit/downloads/pixiv_test.rb +++ b/test/unit/downloads/pixiv_test.rb @@ -4,13 +4,9 @@ module Downloads class PixivTest < ActiveSupport::TestCase context "An ugoira site for pixiv" do setup do - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364", @tempfile.path) - @download.download! - end - - teardown do - @tempfile.unlink + @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @tempfile = @download.download! + @tempfile.close! end should "capture the data" do diff --git a/test/unit/pixiv_ugoira_converter_test.rb b/test/unit/pixiv_ugoira_converter_test.rb index ed7b9a146..bba9e4c58 100644 --- a/test/unit/pixiv_ugoira_converter_test.rb +++ b/test/unit/pixiv_ugoira_converter_test.rb @@ -3,9 +3,7 @@ require "test_helper" class PixivUgoiraConverterTest < ActiveSupport::TestCase context "An ugoira converter" do setup do - @zipped_body = "#{Rails.root}/test/fixtures/ugoira.zip" - @write_file = Tempfile.new("converted") - @preview_write_file = Tempfile.new("preview") + @zipfile = upload_file("test/fixtures/ugoira.zip").tempfile @frame_data = [ {"file" => "000000.jpg", "delay" => 200}, {"file" => "000001.jpg", "delay" => 200}, @@ -15,16 +13,11 @@ class PixivUgoiraConverterTest < ActiveSupport::TestCase ] end - teardown do - @write_file.unlink - @preview_write_file.unlink - end - should "output to webm" do - @converter = PixivUgoiraConverter - @converter.convert(@zipped_body, @write_file.path, @preview_write_file.path, @frame_data) - assert_operator(File.size(@write_file.path), :>, 1_000) - assert_operator(File.size(@preview_write_file.path), :>, 0) + sample_file = PixivUgoiraConverter.generate_webm(@zipfile, @frame_data) + preview_file = PixivUgoiraConverter.generate_preview(@zipfile) + assert_operator(sample_file.size, :>, 1_000) + assert_operator(preview_file.size, :>, 0) end end -end \ No newline at end of file +end diff --git a/test/unit/post_replacement_test.rb b/test/unit/post_replacement_test.rb index 08ecea2e5..8d8df7653 100644 --- a/test/unit/post_replacement_test.rb +++ b/test/unit/post_replacement_test.rb @@ -1,15 +1,6 @@ require 'test_helper' class PostReplacementTest < ActiveSupport::TestCase - def upload_file(path, filename, &block) - Tempfile.open do |file| - file.write(File.read(path)) - file.seek(0) - uploaded_file = ActionDispatch::Http::UploadedFile.new(tempfile: file, filename: filename) - yield uploaded_file - end - end - def setup super @@ -68,7 +59,7 @@ class PostReplacementTest < ActiveSupport::TestCase assert_equal(5969, @post.file_size) assert_equal("png", @post.file_ext) assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", @post.md5) - assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file).hexdigest) assert_equal("https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png", @post.source) end end @@ -102,7 +93,7 @@ class PostReplacementTest < ActiveSupport::TestCase assert_equal(8558, @post.file_size) assert_equal("gif", @post.file_ext) assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5) - assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file).hexdigest) assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source) end @@ -155,18 +146,17 @@ class PostReplacementTest < ActiveSupport::TestCase assert_equal(16275, @post.file_size) assert_equal("png", @post.file_ext) assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5) - assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest) assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source) assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url) end - should "delete the old files after three days" do - old_file_path, old_preview_file_path, old_large_file_path = @post.file_path, @post.preview_file_path, @post.large_file_path + should "delete the old files after thirty days" do + old_file_path, old_preview_file_path = @post.file(:original).path, @post.file(:preview).path @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") assert(File.exists?(old_file_path)) assert(File.exists?(old_preview_file_path)) - assert(File.exists?(old_large_file_path)) Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do Delayed::Worker.new.work_off @@ -174,7 +164,6 @@ class PostReplacementTest < ActiveSupport::TestCase assert_not(File.exists?(old_file_path)) assert_not(File.exists?(old_preview_file_path)) - assert_not(File.exists?(old_large_file_path)) end end @@ -188,7 +177,7 @@ class PostReplacementTest < ActiveSupport::TestCase assert_equal(2804, @post.file_size) assert_equal("zip", @post.file_ext) assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5) - assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest) assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source) assert_equal([{"file"=>"000000.jpg", "delay"=>125}, {"file"=>"000001.jpg", "delay"=>125}], @post.pixiv_ugoira_frame_data.data) @@ -201,17 +190,15 @@ class PostReplacementTest < ActiveSupport::TestCase @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - assert(File.exists?(@post.file_path)) - assert(File.exists?(@post.preview_file_path)) - assert(File.exists?(@post.large_file_path)) + assert_nothing_raised { @post.file(:original) } + assert_nothing_raised { @post.file(:preview) } Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do Delayed::Worker.new.work_off end - assert(File.exists?(@post.file_path)) - assert(File.exists?(@post.preview_file_path)) - assert(File.exists?(@post.large_file_path)) + assert_nothing_raised { @post.file(:original) } + assert_nothing_raised { @post.file(:preview) } end end @@ -231,14 +218,14 @@ class PostReplacementTest < ActiveSupport::TestCase Delayed::Worker.new.work_off end - assert(File.exists?(@post1.file_path)) - assert(File.exists?(@post2.file_path)) + assert_nothing_raised { @post1.file(:original) } + assert_nothing_raised { @post2.file(:original) } end end context "a post with an uploaded file" do should "work" do - upload_file("#{Rails.root}/test/files/test.png", "test.png") do |file| + upload_file("test/files/test.png") do |file| @post.replace!(replacement_file: file, replacement_url: "") assert_equal(@post.md5, Digest::MD5.file(file.tempfile).hexdigest) assert_equal("file://test.png", @post.replacements.last.replacement_url) @@ -268,7 +255,7 @@ class PostReplacementTest < ActiveSupport::TestCase context "a post with the same file" do should "not raise a duplicate error" do - upload_file("#{Rails.root}/test/files/test.jpg", "test.jpg") do |file| + upload_file("test/files/test.jpg") do |file| assert_nothing_raised do @post.replace!(replacement_file: file, replacement_url: "") end @@ -276,7 +263,7 @@ class PostReplacementTest < ActiveSupport::TestCase end should "not queue a deletion or log a comment" do - upload_file("#{Rails.root}/test/files/test.jpg", "test.jpg") do |file| + upload_file("test/files/test.jpg") do |file| assert_no_difference(["@post.comments.count"]) do @post.replace!(replacement_file: file, replacement_url: "") end diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index d805c11fe..a09ba0134 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -35,17 +35,15 @@ class PostTest < ActiveSupport::TestCase end should "delete the files" do - assert_equal(true, File.exists?(@post.preview_file_path)) - assert_equal(true, File.exists?(@post.large_file_path)) - assert_equal(true, File.exists?(@post.file_path)) + assert_nothing_raised { @post.file(:preview) } + assert_nothing_raised { @post.file(:original) } TestAfterCommit.with_commits(true) do @post.expunge! end - assert_equal(false, File.exists?(@post.preview_file_path)) - assert_equal(false, File.exists?(@post.large_file_path)) - assert_equal(false, File.exists?(@post.file_path)) + assert_raise(StandardError) { @post.file(:preview) } + assert_raise(StandardError) { @post.file(:original) } end should "remove all favorites" do diff --git a/test/unit/storage_manager_test.rb b/test/unit/storage_manager_test.rb new file mode 100644 index 000000000..5b4adc911 --- /dev/null +++ b/test/unit/storage_manager_test.rb @@ -0,0 +1,124 @@ +require 'test_helper' + +class StorageManagerTest < ActiveSupport::TestCase + BASE_DIR = "#{Rails.root}/tmp/test-storage" + + setup do + CurrentUser.ip_addr = "127.0.0.1" + end + + context "StorageManager::Local" do + setup do + @storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data") + end + + teardown do + FileUtils.rm_rf(BASE_DIR) + end + + context "#store method" do + should "store the file" do + @storage_manager.store(StringIO.new("data"), "#{BASE_DIR}/test.txt") + + assert("data", File.read("#{BASE_DIR}/test.txt")) + end + + should "overwrite the file if it already exists" do + @storage_manager.store(StringIO.new("foo"), "#{BASE_DIR}/test.txt") + @storage_manager.store(StringIO.new("bar"), "#{BASE_DIR}/test.txt") + + assert("bar", File.read("#{BASE_DIR}/test.txt")) + end + end + + context "#delete method" do + should "delete the file" do + @storage_manager.store(StringIO.new("data"), "test.txt") + @storage_manager.delete("test.txt") + + assert_not(File.exist?("#{BASE_DIR}/test.txt")) + end + + should "not fail if the file doesn't exist" do + assert_nothing_raised { @storage_manager.delete("dne.txt") } + end + end + + context "#store_file and #delete_file methods" do + setup do + @post = FactoryGirl.create(:post, file_ext: "png") + + @storage_manager.store_file(StringIO.new("data"), @post, :preview) + @storage_manager.store_file(StringIO.new("data"), @post, :large) + @storage_manager.store_file(StringIO.new("data"), @post, :original) + + @file_path = "#{BASE_DIR}/preview/#{@post.md5}.jpg" + @large_file_path = "#{BASE_DIR}/sample/sample-#{@post.md5}.jpg" + @preview_file_path = "#{BASE_DIR}/#{@post.md5}.#{@post.file_ext}" + end + + should "store the files at the correct path" do + assert(File.exist?(@file_path)) + assert(File.exist?(@large_file_path)) + assert(File.exist?(@preview_file_path)) + end + + should "delete the files" do + @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :preview) + @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :large) + @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :original) + + assert_not(File.exist?(@file_path)) + assert_not(File.exist?(@large_file_path)) + assert_not(File.exist?(@preview_file_path)) + end + end + + context "#file_url method" do + should "return the correct urls" do + @post = FactoryGirl.create(:post, file_ext: "png") + @storage_manager.stubs(:tagged_filenames).returns(false) + + assert_equal("/data/#{@post.md5}.png", @storage_manager.file_url(@post, :original)) + assert_equal("/data/sample/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large)) + assert_equal("/data/preview/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview)) + end + end + end + + context "StorageManager::Hybrid" do + setup do + @post1 = FactoryGirl.build(:post, id: 1, file_ext: "png") + @post2 = FactoryGirl.build(:post, id: 2, file_ext: "png") + + @storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type| + if id.odd? + StorageManager::Local.new(base_dir: "#{BASE_DIR}/i1", base_url: "/i1") + else + StorageManager::Local.new(base_dir: "#{BASE_DIR}/i2", base_url: "/i2") + end + end + end + + teardown do + FileUtils.rm_rf(BASE_DIR) + end + + context "#store_file method" do + should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do + @storage_manager.store_file(StringIO.new("post1"), @post1, :original) + @storage_manager.store_file(StringIO.new("post2"), @post2, :original) + + assert(File.exist?("#{BASE_DIR}/i1/#{@post1.md5}.png")) + assert(File.exist?("#{BASE_DIR}/i2/#{@post2.md5}.png")) + end + end + + context "#file_url method" do + should "generate /i1 urls for odd posts and /i2 urls for even posts" do + assert_equal("/i1/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original)) + assert_equal("/i2/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original)) + end + end + end +end diff --git a/test/unit/upload_test.rb b/test/unit/upload_test.rb index 022d66e6a..cd7184c6f 100644 --- a/test/unit/upload_test.rb +++ b/test/unit/upload_test.rb @@ -17,21 +17,15 @@ class UploadTest < ActiveSupport::TestCase teardown do CurrentUser.user = nil CurrentUser.ip_addr = nil - - @upload.delete_temp_file if @upload end context "An upload" do - teardown do - FileUtils.rm_f(Dir.glob("#{Rails.root}/tmp/test.*")) - end - context "from a user that is limited" do setup do CurrentUser.user = FactoryGirl.create(:user, :created_at => 1.year.ago) User.any_instance.stubs(:upload_limit).returns(0) end - + should "fail creation" do @upload = FactoryGirl.build(:jpg_upload, :tag_string => "") @upload.save @@ -41,73 +35,51 @@ class UploadTest < ActiveSupport::TestCase context "image size calculator" do should "discover the dimensions for a compressed SWF" do - @upload = FactoryGirl.create(:upload, :file_path => "#{Rails.root}/test/files/compressed.swf") - @upload.calculate_dimensions(@upload.file_path) - assert_equal(607, @upload.image_width) - assert_equal(756, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/compressed.swf")) + assert_equal([607, 756], @upload.calculate_dimensions) end should "discover the dimensions for a JPG with JFIF data" do @upload = FactoryGirl.create(:jpg_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(500, @upload.image_width) - assert_equal(335, @upload.image_height) + assert_equal([500, 335], @upload.calculate_dimensions) end should "discover the dimensions for a JPG with EXIF data" do - @upload = FactoryGirl.create(:exif_jpg_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(529, @upload.image_width) - assert_equal(600, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test-exif-small.jpg")) + assert_equal([529, 600], @upload.calculate_dimensions) end should "discover the dimensions for a JPG with no header data" do - @upload = FactoryGirl.create(:blank_jpg_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(668, @upload.image_width) - assert_equal(996, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test-blank.jpg")) + assert_equal([668, 996], @upload.calculate_dimensions) end should "discover the dimensions for a PNG" do - @upload = FactoryGirl.create(:png_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(768, @upload.image_width) - assert_equal(1024, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.png")) + assert_equal([768, 1024], @upload.calculate_dimensions) end should "discover the dimensions for a GIF" do - @upload = FactoryGirl.create(:gif_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(400, @upload.image_width) - assert_equal(400, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.gif")) + assert_equal([400, 400], @upload.calculate_dimensions) end end context "content type calculator" do should "know how to parse jpeg, png, gif, and swf file headers" do - @upload = FactoryGirl.create(:jpg_upload) - assert_equal("image/jpeg", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.jpg")) - assert_equal("image/gif", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.gif")) - assert_equal("image/png", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.png")) - assert_equal("application/x-shockwave-flash", @upload.file_header_to_content_type("#{Rails.root}/test/files/compressed.swf")) - assert_equal("application/octet-stream", @upload.file_header_to_content_type("#{Rails.root}/README.md")) - end - - should "know how to parse jpeg, png, gif, and swf content types" do - @upload = FactoryGirl.create(:jpg_upload) - assert_equal("jpg", @upload.content_type_to_file_ext("image/jpeg")) - assert_equal("gif", @upload.content_type_to_file_ext("image/gif")) - assert_equal("png", @upload.content_type_to_file_ext("image/png")) - assert_equal("swf", @upload.content_type_to_file_ext("application/x-shockwave-flash")) - assert_equal("bin", @upload.content_type_to_file_ext("")) + @upload = FactoryGirl.build(:jpg_upload) + assert_equal("jpg", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.jpg"))) + assert_equal("gif", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.gif"))) + assert_equal("png", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.png"))) + assert_equal("swf", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/compressed.swf"))) + assert_equal("bin", @upload.file_header_to_file_ext(File.open("#{Rails.root}/README.md"))) end end context "downloader" do context "for a zip that is not an ugoira" do should "not validate" do - FileUtils.cp("#{Rails.root}/test/files/invalid_ugoira.zip", "#{Rails.root}/tmp") - @upload = Upload.create(:file => upload_zip("#{Rails.root}/tmp/invalid_ugoira.zip"), :rating => "q", :tag_string => "xxx") + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/invalid_ugoira.zip")) @upload.process! assert_equal("error: RuntimeError - missing frame data for ugoira", @upload.status) end @@ -116,30 +88,15 @@ class UploadTest < ActiveSupport::TestCase context "that is a pixiv ugoira" do setup do @url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654" - @upload = FactoryGirl.create(:source_upload, :source => @url, :tag_string => "ugoira") - @output_file = Tempfile.new("download") + @upload = FactoryGirl.create(:upload, :source => @url, :tag_string => "ugoira") end - teardown do - @output_file.unlink - end - should "process successfully" do - @upload.download_from_source(@output_file.path) - assert_operator(File.size(@output_file.path), :>, 1_000) - assert_equal("application/zip", @upload.file_header_to_content_type(@output_file.path)) - assert_equal("zip", @upload.content_type_to_file_ext(@upload.file_header_to_content_type(@output_file.path))) + _, _, output_file = @upload.download_from_source(@url, "") + assert_operator(output_file.size, :>, 1_000) + assert_equal("zip", @upload.file_header_to_file_ext(output_file)) end end - - should "initialize the final path after downloading a file" do - @upload = FactoryGirl.create(:source_upload) - path = "#{Rails.root}/tmp/test.download.jpg" - assert_nothing_raised {@upload.download_from_source(path)} - assert(File.exists?(path)) - assert_equal(8558, File.size(path)) - assert_equal(path, @upload.file_path) - end end context "determining if a file is downloadable" do @@ -161,46 +118,32 @@ class UploadTest < ActiveSupport::TestCase context "file processor" do should "parse and process a cgi file representation" do - FileUtils.cp("#{Rails.root}/test/files/test.jpg", "#{Rails.root}/tmp") - @upload = Upload.new(:file => upload_jpeg("#{Rails.root}/tmp/test.jpg")) - assert_nothing_raised {@upload.convert_cgi_file} - assert(File.exists?(@upload.file_path)) - assert_equal(28086, File.size(@upload.file_path)) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.jpg")) + assert_nothing_raised {@upload.process_upload} + assert_equal(28086, @upload.file_size) end should "process a transparent png" do - FileUtils.cp("#{Rails.root}/test/files/alpha.png", "#{Rails.root}/tmp") - @upload = Upload.new(:file => upload_file("#{Rails.root}/tmp/alpha.png", "image/png", "alpha.png")) - assert_nothing_raised {@upload.convert_cgi_file} - assert(File.exists?(@upload.file_path)) - assert_equal(1136, File.size(@upload.file_path)) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/alpha.png")) + assert_nothing_raised {@upload.process_upload} + assert_equal(1136, @upload.file_size) end end context "hash calculator" do should "caculate the hash" do @upload = FactoryGirl.create(:jpg_upload) - @upload.calculate_hash(@upload.file_path) + @upload.process_upload assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5) end end context "resizer" do - teardown do - FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/preview/test.*.jpg")) - FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/sample/test.*.jpg")) - FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/test.*.jpg")) - end - should "generate several resized versions of the image" do - @upload = FactoryGirl.create(:large_jpg_upload) - @upload.calculate_hash(@upload.file_path) - @upload.calculate_dimensions(@upload.file_path) - assert_nothing_raised {@upload.generate_resizes(@upload.file_path)} - assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.small_image_width))) - assert(File.size(@upload.resized_file_path_for(Danbooru.config.small_image_width)) > 0) - assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.large_image_width))) - assert(File.size(@upload.resized_file_path_for(Danbooru.config.large_image_width)) > 0) + @upload = FactoryGirl.create(:upload, file_ext: "jpg", image_width: 1356, image_height: 911, file: upload_file("test/files/test-large.jpg")) + preview_file, sample_file = @upload.generate_resizes + assert_operator(preview_file.size, :>, 1_000) + assert_operator(sample_file.size, :>, 1_000) end end @@ -215,13 +158,10 @@ class UploadTest < ActiveSupport::TestCase context "with an artist commentary" do setup do @upload = FactoryGirl.create(:source_upload, - :rating => "s", - :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" - ) - @upload.include_artist_commentary = "1" - @upload.artist_commentary_title = "" - @upload.artist_commentary_desc = "blah" + include_artist_commentary: "1", + artist_commentary_title: "", + artist_commentary_desc: "blah", + ) end should "create an artist commentary when processed" do @@ -255,15 +195,17 @@ class UploadTest < ActiveSupport::TestCase assert_equal(post.id, @upload.post_id) assert_equal("completed", @upload.status) end + + context "automatic tagging" do + should "tag animated png files" do + @upload = FactoryGirl.build(:upload, file_ext: "png", file: upload_file("test/files/apng/normal_apng.png")) + assert_equal("animated_png", @upload.automatic_tags) + end + end end should "process completely for a pixiv ugoira" do - @upload = FactoryGirl.create(:source_upload, - :source => "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654", - :rating => "s", - :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" - ) + @upload = FactoryGirl.create(:source_upload, source: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654") assert_difference(["PixivUgoiraFrameData.count", "Post.count"]) do @upload.process! assert_equal([], @upload.errors.full_messages) @@ -274,18 +216,18 @@ class UploadTest < ActiveSupport::TestCase assert_equal(60, post.image_width) assert_equal(60, post.image_height) assert_equal("https://i.pximg.net/img-zip-ugoira/img/2014/10/05/23/42/23/46378654_ugoira1920x1080.zip", post.source) - assert_operator(File.size(post.large_file_path), :>, 0) - assert_operator(File.size(post.preview_file_path), :>, 0) + assert_nothing_raised { post.file(:original) } + assert_nothing_raised { post.file(:large) } + assert_nothing_raised { post.file(:preview) } end should "process completely for an uploaded image" do @upload = FactoryGirl.create(:jpg_upload, :rating => "s", :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" + :tag_string => "hoge foo", + :file => upload_file("test/files/test.jpg"), ) - @upload.file = upload_jpeg("#{Rails.root}/test/files/test.jpg") - @upload.convert_cgi_file assert_difference("Post.count") do assert_nothing_raised {@upload.process!} @@ -297,8 +239,8 @@ class UploadTest < ActiveSupport::TestCase assert_equal("127.0.0.1", post.uploader_ip_addr.to_s) assert_equal(@upload.md5, post.md5) assert_equal("jpg", post.file_ext) - assert(File.exists?(post.file_path)) - assert_equal(28086, File.size(post.file_path)) + assert_nothing_raised { post.file(:original) } + assert_equal(28086, post.file(:original).size) assert_equal(post.id, @upload.post_id) assert_equal("completed", @upload.status) end @@ -310,16 +252,5 @@ class UploadTest < ActiveSupport::TestCase assert_nothing_raised {@upload.process!} end end - - should "delete the temporary file upon completion" do - @upload = FactoryGirl.create(:source_upload, - :rating => "s", - :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" - ) - - @upload.process! - assert(!File.exists?(@upload.temp_file_path)) - end end end