From b0c7d9c18527be41ab79b8db9c30c5522d1f684e Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 14 Mar 2018 16:57:29 -0500 Subject: [PATCH 01/10] Add storage managers (local, sftp, s3, hybrid). --- app/logical/storage_manager.rb | 106 ++++++++++++++++++++++++++ app/logical/storage_manager/hybrid.rb | 23 ++++++ app/logical/storage_manager/local.rb | 25 ++++++ app/logical/storage_manager/null.rb | 13 ++++ app/logical/storage_manager/s3.rb | 43 +++++++++++ app/logical/storage_manager/sftp.rb | 76 ++++++++++++++++++ config/danbooru_default_config.rb | 43 +++++++++++ test/test_helper.rb | 3 + 8 files changed, 332 insertions(+) create mode 100644 app/logical/storage_manager.rb create mode 100644 app/logical/storage_manager/hybrid.rb create mode 100644 app/logical/storage_manager/local.rb create mode 100644 app/logical/storage_manager/null.rb create mode 100644 app/logical/storage_manager/s3.rb create mode 100644 app/logical/storage_manager/sftp.rb diff --git a/app/logical/storage_manager.rb b/app/logical/storage_manager.rb new file mode 100644 index 000000000..1ce833f88 --- /dev/null +++ b/app/logical/storage_manager.rb @@ -0,0 +1,106 @@ +class StorageManager + class Error < StandardError; end + + DEFAULT_BASE_URL = Rails.application.routes.url_helpers.root_url + "data" + DEFAULT_BASE_DIR = "#{Rails.root}/public/data" + + attr_reader :base_url, :base_dir, :hierarchical, :tagged_filenames, :large_image_prefix + + def initialize(base_url: DEFAULT_BASE_URL, base_dir: DEFAULT_BASE_DIR, hierarchical: false, tagged_filenames: Danbooru.config.enable_seo_post_urls, large_image_prefix: Danbooru.config.large_image_prefix) + @base_url = base_url.chomp("/") + @base_dir = base_dir + @hierarchical = hierarchical + @tagged_filenames = tagged_filenames + @large_image_prefix = large_image_prefix + end + + # Store the given file at the given path. If a file already exists at that + # location it should be overwritten atomically. Either the file is fully + # written, or an error is raised and the original file is left unchanged. The + # file should never be in a partially written state. + def store(io, path) + raise NotImplementedError, "store not implemented" + end + + # Delete the file at the given path. If the file doesn't exist, no error + # should be raised. + def delete(path) + raise NotImplementedError, "delete not implemented" + end + + # Return a readonly copy of the file located at the given path. + def open(path) + raise NotImplementedError, "open not implemented" + end + + def store_file(io, post, type) + store(io, file_path(post.md5, post.file_ext, type)) + end + + def delete_file(post_id, md5, file_ext, type) + delete(file_path(md5, file_ext, type)) + end + + def open_file(post, type) + open(file_path(post.md5, post.file_ext, type)) + end + + def file_url(post, type) + subdir = subdir_for(post.md5) + file = file_name(post.md5, post.file_ext, type) + + if type == :preview && !post.has_preview? + "#{base_url}/images/download-preview.png" + elsif type == :preview + "#{base_url}/preview/#{subdir}#{file}" + elsif type == :large && post.has_large? + "#{base_url}/sample/#{subdir}#{seo_tags(post)}#{file}" + else + "#{base_url}/#{subdir}#{seo_tags(post)}#{file}" + end + end + + protected + + def file_path(md5, file_ext, type) + subdir = subdir_for(md5) + file = file_name(md5, file_ext, type) + + case type + when :preview + "#{base_dir}/preview/#{subdir}#{file}" + when :large + "#{base_dir}/sample/#{subdir}#{file}" + when :original + "#{base_dir}/#{subdir}#{file}" + end + end + + def file_name(md5, file_ext, type) + large_file_ext = (file_ext == "zip") ? "webm" : "jpg" + + case type + when :preview + "#{md5}.jpg" + when :large + "#{large_image_prefix}#{md5}.#{large_file_ext}" + when :original + "#{md5}.#{file_ext}" + end + end + + def subdir_for(md5) + if hierarchical + "#{md5[0..1]}/#{md5[2..3]}/" + else + "" + end + end + + def seo_tags(post, user = CurrentUser.user) + return "" if !tagged_filenames || user.disable_tagged_filenames? + + tags = post.humanized_essential_tag_string.gsub(/[^a-z0-9]+/, "_").gsub(/(?:^_+)|(?:_+$)/, "").gsub(/_{2,}/, "_") + "__#{tags}__" + end +end diff --git a/app/logical/storage_manager/hybrid.rb b/app/logical/storage_manager/hybrid.rb new file mode 100644 index 000000000..1e3afdda4 --- /dev/null +++ b/app/logical/storage_manager/hybrid.rb @@ -0,0 +1,23 @@ +class StorageManager::Hybrid < StorageManager + attr_reader :submanager + + def initialize(&block) + @submanager = block + end + + def store_file(io, post, type) + submanager[post.id, post.md5, post.file_ext, type].store_file(io, post, type) + end + + def delete_file(post_id, md5, file_ext, type) + submanager[post_id, md5, file_ext, type].delete_file(post_id, md5, file_ext, type) + end + + def open_file(io, post, type) + submanager[post.id, post.md5, post.file_ext, type].open_file(post, type) + end + + def file_url(post, type) + submanager[post.id, post.md5, post.file_ext, type].file_url(post, type) + end +end diff --git a/app/logical/storage_manager/local.rb b/app/logical/storage_manager/local.rb new file mode 100644 index 000000000..6bdfa69e3 --- /dev/null +++ b/app/logical/storage_manager/local.rb @@ -0,0 +1,25 @@ +class StorageManager::Local < StorageManager + DEFAULT_PERMISSIONS = 0644 + + def store(io, dest_path) + temp_path = dest_path + "-" + SecureRandom.uuid + ".tmp" + + FileUtils.mkdir_p(File.dirname(temp_path)) + bytes_copied = IO.copy_stream(io, temp_path) + raise Error, "store failed: #{bytes_copied}/#{io.size} bytes copied" if bytes_copied != io.size + + FileUtils.chmod(DEFAULT_PERMISSIONS, temp_path) + File.rename(temp_path, dest_path) + rescue StandardError => e + FileUtils.rm_f(temp_path) + raise Error, e + end + + def delete(path) + FileUtils.rm_f(path) + end + + def open(path) + File.open(path, "r", binmode: true) + end +end diff --git a/app/logical/storage_manager/null.rb b/app/logical/storage_manager/null.rb new file mode 100644 index 000000000..ebaeb8bfe --- /dev/null +++ b/app/logical/storage_manager/null.rb @@ -0,0 +1,13 @@ +class StorageManager::Null < StorageManager + def store(io, path) + # no-op + end + + def delete(path) + # no-op + end + + def open(path) + # no-op + end +end diff --git a/app/logical/storage_manager/s3.rb b/app/logical/storage_manager/s3.rb new file mode 100644 index 000000000..6549bf0bf --- /dev/null +++ b/app/logical/storage_manager/s3.rb @@ -0,0 +1,43 @@ +class StorageManager::S3 < StorageManager + # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method + DEFAULT_S3_OPTIONS = { + region: Danbooru.config.aws_region, + credentials: Danbooru.config.aws_credentials, + logger: Rails.logger, + } + + # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#put_object-instance_method + DEFAULT_PUT_OPTIONS = { + acl: "public-read", + storage_class: "STANDARD", # STANDARD, STANDARD_IA, REDUCED_REDUNDANCY + cache_control: "public, max-age=#{1.year.to_i}", + #content_type: "image/jpeg" # XXX should set content type + } + + attr_reader :bucket, :client, :s3_options + + def initialize(bucket, client: nil, s3_options: {}, **options) + @bucket = bucket + @s3_options = DEFAULT_S3_OPTIONS.merge(s3_options) + @client = client || Aws::S3::Client.new(**@s3_options) + super(**options) + end + + def store(io, path) + data = io.read + base64_md5 = Digest::MD5.base64digest(data) + client.put_object(bucket: bucket, key: path, body: data, content_md5: base64_md5, **DEFAULT_PUT_OPTIONS) + end + + def delete(path) + client.delete_object(bucket: bucket, key: path) + rescue Aws::S3::Errors::NoSuchKey + # ignore + end + + def open(path) + file = Tempfile.new(binmode: true) + client.get_object(bucket: bucket: key: path, response_target: file) + file + end +end diff --git a/app/logical/storage_manager/sftp.rb b/app/logical/storage_manager/sftp.rb new file mode 100644 index 000000000..add10d010 --- /dev/null +++ b/app/logical/storage_manager/sftp.rb @@ -0,0 +1,76 @@ +class StorageManager::SFTP < StorageManager + DEFAULT_PERMISSIONS = 0644 + + # http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start + DEFAULT_SSH_OPTIONS = { + timeout: 10, + logger: Rails.logger, + verbose: :fatal, + non_interactive: true, + } + + attr_reader :hosts, :ssh_options + + def initialize(*hosts, ssh_options: {}, **options) + @hosts = hosts + @ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options) + super(**options) + end + + def store(file, dest_path) + temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp" + dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak" + + each_host do |host, sftp| + begin + sftp.upload!(file.path, temp_upload_path) + sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS) + + # `rename!` can't overwrite existing files, so if a file already exists + # at dest_path we move it out of the way first. + force { sftp.rename!(dest_path, dest_backup_path) } + force { sftp.rename!(temp_upload_path, dest_path) } + rescue StandardError => e + # if anything fails, try to move the original file back in place (if it was moved). + force { sftp.rename!(dest_backup_path, dest_path) } + raise Error, e + ensure + force { sftp.remove!(temp_upload_path) } + force { sftp.remove!(dest_backup_path) } + end + end + end + + def delete(dest_path) + each_host do |host, sftp| + force { sftp.remove!(dest_path) } + end + end + + def open(dest_path) + file = Tempfile.new(binmode: true) + + Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp| + sftp.download!(dest_path, file.path) + end + + file + end + + protected + + # Ignore "no such file" exceptions for the given operation. + def force + yield + rescue Net::SFTP::StatusException => e + raise Error, e unless e.description == "no such file" + end + + def each_host + hosts.each do |host| + Net::SFTP.start(host, nil, ssh_options) do |sftp| + yield host, sftp + end + end + end +end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index fd30ddc0c..a1bf74ff6 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -222,6 +222,41 @@ module Danbooru "danbooru" end + # The method to use for storing image files. + def storage_manager + # Store files on the local filesystem. + # base_dir - where to store files (default: under public/data) + # base_url - where to serve files from (default: http://#{hostname}/data) + # hierarchical: false - store files in a single directory + # hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash + StorageManager::Local.new(base_dir: "#{Rails.root}/public/data", hierarchical: false) + + # Store files on one or more remote host(s). Configure SSH settings in + # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) + # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {}) + + # Store files in an S3 bucket. The bucket must already exist and be + # writable by you. Configure your S3 settings in aws_region and + # aws_credentials below, or in the s3_options param (ref: + # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method) + # StorageManager::S3.new("my_s3_bucket", base_url: "https://my_s3_bucket.s3.amazonaws.com/", s3_options: {}) + + # Select the storage method based on the post's id and type (preview, large, or original). + # StorageManager::Hybrid.new do |id, md5, file_ext, type| + # ssh_options = { user: "danbooru" } + # + # if type.in?([:large, :original]) && id.in?(0..850_000) + # StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options) + # elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000) + # StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options) + # elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000) + # StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options) + # else + # StorageManager::SFTP.new(*all_server_hosts, ssh_options: ssh_options) + # end + # end + end + def build_file_url(post) "/data/#{post.file_path_prefix}/#{post.md5}.#{post.file_ext}" end @@ -611,6 +646,14 @@ module Danbooru end # AWS config options + def aws_region + "us-east-1" + end + + def aws_credentials + Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key) + end + def aws_access_key_id end diff --git a/test/test_helper.rb b/test/test_helper.rb index 04a3d3edd..19474ca49 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -38,6 +38,9 @@ class ActiveSupport::TestCase mock_missed_search_service! WebMock.allow_net_connect! Danbooru.config.stubs(:enable_sock_puppet_validation?).returns(false) + + storage_manager = StorageManager::Local.new(base_dir: "#{Rails.root}/public/data/test") + Danbooru.config.stubs(:storage_manager).returns(storage_manager) end teardown do From 2286ccfca8e6889a25014a2224c189b71037f5ea Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 18 Mar 2018 01:59:26 -0500 Subject: [PATCH 02/10] uploads: clean up process_upload. * Remove `initialize_status` (status already defaults to pending in database) * Remove `has_dimensions?` (always returns true) * Remove `async_conversion?` (dead code) * Remove `validate_file_exists` (unneeded checks) * Simplify `calculate_dimensions` * Merge `file_header_to_content_type` with `content_type_to_file_ext` (content type isn't used elsewhere) --- app/models/upload.rb | 110 +++++++------------------------------------ 1 file changed, 18 insertions(+), 92 deletions(-) diff --git a/app/models/upload.rb b/app/models/upload.rb index 8bf5a7872..1446155c2 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -10,7 +10,6 @@ class Upload < ApplicationRecord belongs_to :uploader, :class_name => "User" belongs_to :post before_validation :initialize_uploader, :on => :create - before_validation :initialize_status, :on => :create before_create :convert_cgi_file after_destroy :delete_temp_file validate :uploader_is_not_limited, :on => :create @@ -53,12 +52,6 @@ class Upload < ApplicationRecord end end - def validate_file_exists - unless file_path && File.exists?(file_path) - raise "file does not exist" - end - end - def validate_file_content_type unless is_valid_content_type? raise "invalid content type (only JPEG, PNG, GIF, SWF, MP4, and WebM files are allowed)" @@ -75,12 +68,6 @@ class Upload < ApplicationRecord end end - def validate_md5_confirmation_after_move - if !md5_confirmation.blank? && md5_confirmation != Digest::MD5.file(md5_file_path).hexdigest - raise "md5 mismatch" - end - end - def rating_given if rating.present? return true @@ -112,13 +99,12 @@ class Upload < ApplicationRecord def process_upload CurrentUser.scoped(uploader, uploader_ip_addr) do update_attribute(:status, "processing") - self.source = strip_source + + self.source = source.to_s.strip if is_downloadable? self.downloaded_source, self.source = download_from_source(temp_file_path) end - validate_file_exists - self.content_type = file_header_to_content_type(file_path) - self.file_ext = content_type_to_file_ext(content_type) + self.file_ext = file_header_to_file_ext(file_path) validate_file_content_type calculate_hash(file_path) validate_md5_uniqueness @@ -126,12 +112,9 @@ class Upload < ApplicationRecord tag_audio validate_video_duration calculate_file_size(file_path) - if has_dimensions? - calculate_dimensions(file_path) - end + self.image_width, self.image_height = calculate_dimensions generate_resizes(file_path) move_file - validate_md5_confirmation_after_move save end end @@ -175,10 +158,6 @@ class Upload < ApplicationRecord delete_temp_file end - def async_conversion? - is_ugoira? - end - def ugoira_service @ugoira_service ||= PixivUgoiraService.new end @@ -288,27 +267,17 @@ class Upload < ApplicationRecord module DimensionMethods # Figures out the dimensions of the image. - def calculate_dimensions(file_path) + def calculate_dimensions if is_video? - self.image_width = video.width - self.image_height = video.height + [video.width, video.height] elsif is_ugoira? ugoira_service.calculate_dimensions(file_path) - self.image_width = ugoira_service.width - self.image_height = ugoira_service.height + [ugoira_service.width, ugoira_service.height] else - File.open(file_path, "rb") do |file| - image_size = ImageSpec.new(file) - self.image_width = image_size.width - self.image_height = image_size.height - end + image_size = ImageSpec.new(file_path) + [image_size.width, image_size.height] end end - - # Does this file have image dimensions? - def has_dimensions? - %w(jpg gif png swf webm mp4 zip).include?(file_ext) - end end module ContentTypeMethods @@ -316,61 +285,26 @@ class Upload < ApplicationRecord file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/ end - def content_type_to_file_ext(content_type) - case content_type - when "image/jpeg" + def file_header_to_file_ext(file_path) + case File.read(file_path, 16) + when /^\xff\xd8/n "jpg" - - when "image/gif" + when /^GIF87a/, /^GIF89a/ "gif" - - when "image/png" + when /^\x89PNG\r\n\x1a\n/n "png" - - when "application/x-shockwave-flash" + when /^CWS/, /^FWS/, /^ZWS/ "swf" - - when "video/webm" + when /^\x1a\x45\xdf\xa3/n "webm" - - when "video/mp4" + when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/ "mp4" - - when "application/zip" + when /^PK\x03\x04/ "zip" - else "bin" end end - - def file_header_to_content_type(source_path) - case File.read(source_path, 16) - when /^\xff\xd8/n - "image/jpeg" - - when /^GIF87a/, /^GIF89a/ - "image/gif" - - when /^\x89PNG\r\n\x1a\n/n - "image/png" - - when /^CWS/, /^FWS/, /^ZWS/ - "application/x-shockwave-flash" - - when /^\x1a\x45\xdf\xa3/n - "video/webm" - - when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/ - "video/mp4" - - when /^PK\x03\x04/ - "application/zip" - - else - "application/octet-stream" - end - end end module FilePathMethods @@ -405,10 +339,6 @@ class Upload < ApplicationRecord end module DownloaderMethods - def strip_source - source.to_s.strip - end - # Determines whether the source is downloadable def is_downloadable? source =~ /^https?:\/\// && file_path.blank? @@ -442,10 +372,6 @@ class Upload < ApplicationRecord end module StatusMethods - def initialize_status - self.status = "pending" - end - def is_pending? status == "pending" end From 60dcfbfbdd7db366801c63e7590a8183bc58a39b Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 18 Mar 2018 11:55:24 -0500 Subject: [PATCH 03/10] uploads: autotag animated_gif/png during upload. Move animated_gif / animated_png autotagging to take place during uploading, instead of during tag editing. We can't generally assume the file will be present on the local filesystem after uploading. --- app/logical/apng_inspector.rb | 4 ++-- app/models/post.rb | 27 --------------------------- app/models/upload.rb | 26 +++++++++++++++++++++----- 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/app/logical/apng_inspector.rb b/app/logical/apng_inspector.rb index 30ce02676..dffad8081 100644 --- a/app/logical/apng_inspector.rb +++ b/app/logical/apng_inspector.rb @@ -86,7 +86,7 @@ class APNGInspector end @corrupted = !read_success || actl_corrupted - return !@corrupted + self end def corrupted? @@ -109,4 +109,4 @@ class APNGInspector return framedata.unpack("N".freeze)[0] end -end \ No newline at end of file +end diff --git a/app/models/post.rb b/app/models/post.rb index 4aa4e9649..4528afe37 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -255,24 +255,6 @@ class Post < ApplicationRecord file_ext =~ /jpg|jpeg|gif|png/i end - def is_animated_gif? - if file_ext =~ /gif/i && File.exists?(file_path) - return Magick::Image.ping(file_path).length > 1 - else - return false - end - end - - def is_animated_png? - if file_ext =~ /png/i && File.exists?(file_path) - apng = APNGInspector.new(file_path) - apng.inspect! - return apng.animated? - else - return false - end - end - def is_flash? file_ext =~ /swf/i end @@ -765,7 +747,6 @@ class Post < ApplicationRecord return tags if !Danbooru.config.enable_dimension_autotagging tags -= %w(incredibly_absurdres absurdres highres lowres huge_filesize flash webm mp4) - tags -= %w(animated_gif animated_png) if new_record? if has_dimensions? if image_width >= 10_000 || image_height >= 10_000 @@ -794,14 +775,6 @@ class Post < ApplicationRecord tags << "huge_filesize" end - if is_animated_gif? - tags << "animated_gif" - end - - if is_animated_png? - tags << "animated_png" - end - if is_flash? tags << "flash" end diff --git a/app/models/upload.rb b/app/models/upload.rb index 1446155c2..a46984a9a 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -80,10 +80,14 @@ class Upload < ApplicationRecord end end - def tag_audio - if is_video? && video.audio_channels.present? - self.tag_string = "#{tag_string} video_with_sound" - end + def automatic_tags + return "" unless Danbooru.config.enable_dimension_autotagging + + tags = [] + tags << "video_with_sound" if is_video_with_audio? + tags << "animated_gif" if is_animated_gif? + tags << "animated_png" if is_animated_png? + tags.join(" ") end def validate_video_duration @@ -109,9 +113,9 @@ class Upload < ApplicationRecord calculate_hash(file_path) validate_md5_uniqueness validate_md5_confirmation - tag_audio validate_video_duration calculate_file_size(file_path) + self.tag_string = "#{tag_string} #{automatic_tags}" self.image_width, self.image_height = calculate_dimensions generate_resizes(file_path) move_file @@ -219,9 +223,21 @@ class Upload < ApplicationRecord %w(webm mp4).include?(file_ext) end + def is_video_with_audio? + is_video? && video.audio_channels.present? + end + def is_ugoira? %w(zip).include?(file_ext) end + + def is_animated_gif? + file_ext == "gif" && Magick::Image.ping(file_path).length > 1 + end + + def is_animated_png? + file_ext == "png" && APNGInspector.new(file_path).inspect!.animated? + end end module ResizerMethods From c76463f34df003c8745a2d1410c1f1c208882356 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 18 Mar 2018 12:05:25 -0500 Subject: [PATCH 04/10] uploads: use storage manager to distribute files. Refactors the upload process to pass around temp files, rather than passing around file paths and directly writing output to the local filesystem. This way we can pass the storage manager the preview / sample / original temp files, so it can deal with storage itself. * Change Download::File#download! to return a temp file. * Change DanbooruImageResizer and PixivUgoiraConverter to accept/return temp files instead of file paths. * Change Upload#generate_resizes to return temp files for previews and samples. * Change Upload#generate_resizes to generate ugoira .webm samples synchronously instead of asynchronously. --- app/controllers/uploads_controller.rb | 2 +- app/logical/danbooru_image_resizer.rb | 9 +- app/logical/downloads/file.rb | 17 ++- app/logical/pixiv_ugoira_converter.rb | 29 +++-- app/logical/pixiv_ugoira_service.rb | 22 ---- app/models/post.rb | 15 +-- app/models/post_replacement.rb | 4 +- app/models/upload.rb | 174 ++++++++------------------ 8 files changed, 89 insertions(+), 183 deletions(-) diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index 055e7e660..55ace2f4e 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -6,7 +6,7 @@ class UploadsController < ApplicationController @upload = Upload.new @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first if params[:url] - download = Downloads::File.new(params[:url], ".") + download = Downloads::File.new(params[:url]) @normalized_url, _, _ = download.before_download(params[:url], {}) @post = find_post_by_url(@normalized_url) diff --git a/app/logical/danbooru_image_resizer.rb b/app/logical/danbooru_image_resizer.rb index 3871acc94..d3697fbc3 100644 --- a/app/logical/danbooru_image_resizer.rb +++ b/app/logical/danbooru_image_resizer.rb @@ -1,6 +1,6 @@ module DanbooruImageResizer - def resize(read_path, write_path, width, height, resize_quality = 90) - image = Magick::Image.read(read_path).first + def resize(file, width, height, resize_quality = 90) + image = Magick::Image.read(file.path).first geometry = "#{width}x>" if width == Danbooru.config.small_image_width @@ -17,14 +17,15 @@ module DanbooruImageResizer image = flatten(image, width, height) image.strip! - image.write(write_path) do + output_file = Tempfile.new(binmode: true) + image.write("jpeg:" + output_file.path) do self.quality = resize_quality # setting PlaneInterlace enables progressive encoding for JPEGs self.interlace = Magick::PlaneInterlace end image.destroy! - FileUtils.chmod(0664, write_path) + output_file end def flatten(image, width, height) diff --git a/app/logical/downloads/file.rb b/app/logical/downloads/file.rb index 911dc21df..ab1cf63ae 100644 --- a/app/logical/downloads/file.rb +++ b/app/logical/downloads/file.rb @@ -3,9 +3,9 @@ module Downloads class Error < Exception ; end attr_reader :data, :options - attr_accessor :source, :original_source, :downloaded_source, :file_path + attr_accessor :source, :original_source, :downloaded_source - def initialize(source, file_path, options = {}) + def initialize(source, options = {}) # source can potentially get rewritten in the course # of downloading a file, so check it again @source = source @@ -14,9 +14,6 @@ module Downloads # the URL actually downloaded after rewriting the original source. @downloaded_source = nil - # where to save the download - @file_path = file_path - # we sometimes need to capture data from the source page @data = {} @@ -35,12 +32,13 @@ module Downloads def download! url, headers, @data = before_download(@source, @data) - ::File.open(@file_path, "wb") do |out| - http_get_streaming(uncached_url(url, headers), out, headers) - end + output_file = Tempfile.new(binmode: true) + http_get_streaming(uncached_url(url, headers), output_file, headers) @downloaded_source = url @source = after_download(url) + + output_file end def before_download(url, datums) @@ -91,7 +89,8 @@ module Downloads end if res.success? - return + file.rewind + return file else raise Error.new("HTTP error code: #{res.code} #{res.message}") end diff --git a/app/logical/pixiv_ugoira_converter.rb b/app/logical/pixiv_ugoira_converter.rb index abe619656..791e14642 100644 --- a/app/logical/pixiv_ugoira_converter.rb +++ b/app/logical/pixiv_ugoira_converter.rb @@ -1,13 +1,9 @@ class PixivUgoiraConverter - def self.convert(source_path, output_path, preview_path, frame_data) - folder = Zip::File.new(source_path) - write_webm(folder, output_path, frame_data) - write_preview(folder, preview_path) - RemoteFileManager.new(output_path).distribute - RemoteFileManager.new(preview_path).distribute - end + def self.generate_webm(ugoira_file, frame_data) + folder = Zip::File.new(ugoira_file.path) + output_file = Tempfile.new(binmode: true) + write_path = output_file.path - def self.write_webm(folder, write_path, frame_data) Dir.mktmpdir do |tmpdir| FileUtils.mkdir_p("#{tmpdir}/images") folder.each_with_index do |file, i| @@ -64,14 +60,17 @@ class PixivUgoiraConverter return end end + + output_file end - def self.write_preview(folder, path) - Dir.mktmpdir do |tmpdir| - file = folder.first - temp_path = File.join(tmpdir, file.name) - file.extract(temp_path) - DanbooruImageResizer.resize(temp_path, path, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - end + def self.generate_preview(ugoira_file) + file = Tempfile.new(binmode: true) + zipfile = Zip::File.new(ugoira_file.path) + zipfile.entries.first.extract(file.path) { true } # 'true' means overwrite the existing tempfile. + + DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) + ensure + file.close! end end diff --git a/app/logical/pixiv_ugoira_service.rb b/app/logical/pixiv_ugoira_service.rb index 248428a8d..58536fa0d 100644 --- a/app/logical/pixiv_ugoira_service.rb +++ b/app/logical/pixiv_ugoira_service.rb @@ -1,32 +1,10 @@ class PixivUgoiraService attr_reader :width, :height, :frame_data, :content_type - def self.regen(post) - service = new() - service.load( - :is_ugoira => true, - :ugoira_frame_data => post.pixiv_ugoira_frame_data.data - ) - service.generate_resizes(post.file_path, post.large_file_path, post.preview_file_path, false) - end - def save_frame_data(post) PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id) end - def generate_resizes(source_path, output_path, preview_path, delay = true) - # Run this a bit in the future to give the upload process time to move the file - if delay - PixivUgoiraConverter.delay(:queue => Socket.gethostname, :run_at => 10.seconds.from_now, :priority => -1).convert(source_path, output_path, preview_path, @frame_data) - else - PixivUgoiraConverter.convert(source_path, output_path, preview_path, @frame_data) - end - - # since the resizes will be delayed, just touch the output file so the - # file distribution wont break - FileUtils.touch([output_path, preview_path]) - end - def calculate_dimensions(source_path) folder = Zip::File.new(source_path) tempfile = Tempfile.new("ugoira-dimensions") diff --git a/app/models/post.rb b/app/models/post.rb index 4528afe37..6b08b1d5b 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -130,17 +130,10 @@ class Post < ApplicationRecord Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true) end - def distribute_files - if Danbooru.config.build_file_url(self) =~ /^http/ - # this post is archived - RemoteFileManager.new(file_path).distribute_to_archive(Danbooru.config.build_file_url(self)) - RemoteFileManager.new(preview_file_path).distribute if has_preview? - RemoteFileManager.new(large_file_path).distribute_to_archive(Danbooru.config.build_large_file_url(self)) if has_large? - else - RemoteFileManager.new(file_path).distribute - RemoteFileManager.new(preview_file_path).distribute if has_preview? - RemoteFileManager.new(large_file_path).distribute if has_large? - end + def distribute_files(file, sample_file, preview_file) + storage_manager.store_file(file, self, :original) + storage_manager.store_file(sample_file, self, :large) if sample_file.present? + storage_manager.store_file(preview_file, self, :preview) if preview_file.present? end def file_path_prefix diff --git a/app/models/post_replacement.rb b/app/models/post_replacement.rb index aefe2715c..b6b3fd11b 100644 --- a/app/models/post_replacement.rb +++ b/app/models/post_replacement.rb @@ -24,6 +24,8 @@ class PostReplacement < ApplicationRecord end def process! + upload = nil + transaction do upload = Upload.create!( file: replacement_file, @@ -79,7 +81,7 @@ class PostReplacement < ApplicationRecord # point of no return: these things can't be rolled back, so we do them # only after the transaction successfully commits. - post.distribute_files + upload.distribute_files(post) post.update_iqdb_async end diff --git a/app/models/upload.rb b/app/models/upload.rb index a46984a9a..6b241c4d4 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -10,13 +10,11 @@ class Upload < ApplicationRecord belongs_to :uploader, :class_name => "User" belongs_to :post before_validation :initialize_uploader, :on => :create - before_create :convert_cgi_file - after_destroy :delete_temp_file validate :uploader_is_not_limited, :on => :create validate :file_or_source_is_present, :on => :create validate :rating_given attr_accessible :file, :image_width, :image_height, :file_ext, :md5, - :file_size, :as_pending, :source, :file_path, :content_type, :rating, + :file_size, :as_pending, :source, :rating, :tag_string, :status, :backtrace, :post_id, :md5_confirmation, :parent_id, :server, :artist_commentary_title, :artist_commentary_desc, :include_artist_commentary, @@ -101,31 +99,36 @@ class Upload < ApplicationRecord module ConversionMethods def process_upload - CurrentUser.scoped(uploader, uploader_ip_addr) do + begin update_attribute(:status, "processing") self.source = source.to_s.strip if is_downloadable? - self.downloaded_source, self.source = download_from_source(temp_file_path) + self.downloaded_source, self.source, self.file = download_from_source(source, referer_url) + else + self.file = self.file.tempfile end - self.file_ext = file_header_to_file_ext(file_path) + + self.file_ext = file_header_to_file_ext(file) + self.file_size = file.size + self.md5 = Digest::MD5.file(file.path).hexdigest + validate_file_content_type - calculate_hash(file_path) validate_md5_uniqueness validate_md5_confirmation validate_video_duration - calculate_file_size(file_path) + self.tag_string = "#{tag_string} #{automatic_tags}" self.image_width, self.image_height = calculate_dimensions - generate_resizes(file_path) - move_file + save end end def create_post_from_upload post = convert_to_post - post.distribute_files + distribute_files(post) + if post.save create_artist_commentary(post) if include_artist_commentary? ugoira_service.save_frame_data(post) if is_ugoira? @@ -138,6 +141,14 @@ class Upload < ApplicationRecord post end + def distribute_files(post) + preview_file, sample_file = generate_resizes + post.distribute_files(file, sample_file, preview_file) + ensure + preview_file.try(:close!) + sample_file.try(:close!) + end + def process!(force = false) @tries ||= 0 return if !force && status =~ /processing|completed|error/ @@ -157,9 +168,9 @@ class Upload < ApplicationRecord rescue Exception => x update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n")) nil - + ensure - delete_temp_file + file.try(:close!) end def ugoira_service @@ -194,23 +205,6 @@ class Upload < ApplicationRecord end module FileMethods - def delete_temp_file(path = nil) - FileUtils.rm_f(path || temp_file_path) - end - - def move_file - FileUtils.mv(file_path, md5_file_path) - end - - def calculate_file_size(source_path) - self.file_size = File.size(source_path) - end - - # Calculates the MD5 based on whatever is in temp_file_path - def calculate_hash(source_path) - self.md5 = Digest::MD5.file(source_path).hexdigest - end - def is_image? %w(jpg gif png).include?(file_ext) end @@ -232,52 +226,43 @@ class Upload < ApplicationRecord end def is_animated_gif? - file_ext == "gif" && Magick::Image.ping(file_path).length > 1 + file_ext == "gif" && Magick::Image.ping(file.path).length > 1 end def is_animated_png? - file_ext == "png" && APNGInspector.new(file_path).inspect!.animated? + file_ext == "png" && APNGInspector.new(file.path).inspect!.animated? end end module ResizerMethods - def generate_resizes(source_path) - generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85) + def generate_resizes + if is_video? + preview_file = generate_video_preview_for(video, width, height) + elsif is_ugoira? + preview_file = PixivUgoiraConverter.generate_preview(file) + sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data) + elsif is_image? + preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - if is_image? && image_width > Danbooru.config.large_image_width - generate_resize_for(Danbooru.config.large_image_width, nil, source_path) + if image_width > Danbooru.config.large_image_width + sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, nil, 90) + end end + + [preview_file, sample_file] end - def generate_video_preview_for(width, height, output_path) - dimension_ratio = image_width.to_f / image_height + def generate_video_preview_for(video, width, height) + dimension_ratio = video.width.to_f / video.height if dimension_ratio > 1 height = (width / dimension_ratio).to_i else width = (height * dimension_ratio).to_i end - video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) - FileUtils.chmod(0664, output_path) - end - def generate_resize_for(width, height, source_path, quality = 90) - unless File.exists?(source_path) - raise Error.new("file not found") - end - - output_path = resized_file_path_for(width) - if is_image? - DanbooruImageResizer.resize(source_path, output_path, width, height, quality) - elsif is_ugoira? - if Delayed::Worker.delay_jobs - # by the time this runs we'll have moved source_path to md5_file_path - ugoira_service.generate_resizes(md5_file_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width)) - else - ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width), false) - end - elsif is_video? - generate_video_preview_for(width, height, output_path) - end + output_file = Tempfile.new(binmode: true) + video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) + output_file end end @@ -287,10 +272,10 @@ class Upload < ApplicationRecord if is_video? [video.width, video.height] elsif is_ugoira? - ugoira_service.calculate_dimensions(file_path) + ugoira_service.calculate_dimensions(file.path) [ugoira_service.width, ugoira_service.height] else - image_size = ImageSpec.new(file_path) + image_size = ImageSpec.new(file.path) [image_size.width, image_size.height] end end @@ -301,8 +286,8 @@ class Upload < ApplicationRecord file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/ end - def file_header_to_file_ext(file_path) - case File.read(file_path, 16) + def file_header_to_file_ext(file) + case File.read(file.path, 16) when /^\xff\xd8/n "jpg" when /^GIF87a/, /^GIF89a/ @@ -323,67 +308,18 @@ class Upload < ApplicationRecord end end - module FilePathMethods - def md5_file_path - prefix = Rails.env == "test" ? "test." : "" - "#{Rails.root}/public/data/#{prefix}#{md5}.#{file_ext}" - end - - def resized_file_path_for(width) - prefix = Rails.env == "test" ? "test." : "" - - case width - when Danbooru.config.small_image_width - "#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg" - - when Danbooru.config.large_image_width - "#{Rails.root}/public/data/sample/#{prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}" - end - end - - def large_file_ext - if is_ugoira? - "webm" - else - "jpg" - end - end - - def temp_file_path - @temp_file_path ||= File.join(Rails.root, "tmp", "upload_#{Time.now.to_f}.#{Process.pid}") - end - end - module DownloaderMethods # Determines whether the source is downloadable def is_downloadable? - source =~ /^https?:\/\// && file_path.blank? + source =~ /^https?:\/\// && file.blank? end - # Downloads the file to destination_path - def download_from_source(destination_path) - self.file_path = destination_path - download = Downloads::File.new(source, destination_path, :referer_url => referer_url) - download.download! + def download_from_source(source, referer_url) + download = Downloads::File.new(source, referer_url: referer_url) + file = download.download! ugoira_service.load(download.data) - [download.downloaded_source, download.source] - end - end - module CgiFileMethods - def convert_cgi_file - return if file.blank? || file.size == 0 - - self.file_path = temp_file_path - - if file.respond_to?(:tempfile) && file.tempfile - FileUtils.cp(file.tempfile.path, file_path) - else - File.open(file_path, 'wb') do |out| - out.write(file.read) - end - end - FileUtils.chmod(0664, file_path) + [download.downloaded_source, download.source, file] end end @@ -422,7 +358,7 @@ class Upload < ApplicationRecord module VideoMethods def video - @video ||= FFMPEG::Movie.new(file_path) + @video ||= FFMPEG::Movie.new(file.path) end end @@ -476,8 +412,6 @@ class Upload < ApplicationRecord include DimensionMethods include ContentTypeMethods include DownloaderMethods - include FilePathMethods - include CgiFileMethods include StatusMethods include UploaderMethods include VideoMethods From 6d0d1a3ce9b6fff89a1d4f89284f3908672a3199 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 19 Mar 2018 23:24:32 -0500 Subject: [PATCH 05/10] posts: use storage manager to build file urls. --- app/models/post.rb | 46 +++++++++---------------------- config/danbooru_default_config.rb | 7 ----- 2 files changed, 13 insertions(+), 40 deletions(-) diff --git a/app/models/post.rb b/app/models/post.rb index 6b08b1d5b..68eabfc55 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -168,8 +168,16 @@ class Post < ApplicationRecord "#{file_path_prefix}#{md5}.#{file_ext}" end + def storage_manager + Danbooru.config.storage_manager + end + + def file(type = :original) + storage_manager.open_file(self, type) + end + def file_url - Danbooru.config.build_file_url(self) + storage_manager.file_url(self, :original) end # this is for the 640x320 version @@ -177,31 +185,11 @@ class Post < ApplicationRecord end def large_file_url - if has_large? - Danbooru.config.build_large_file_url(self) - else - file_url - end - end - - def seo_tag_string - if Danbooru.config.enable_seo_post_urls && !CurrentUser.user.disable_tagged_filenames? - "__#{seo_tags}__" - else - nil - end - end - - def seo_tags - @seo_tags ||= humanized_essential_tag_string.gsub(/[^a-z0-9]+/, "_").gsub(/(?:^_+)|(?:_+$)/, "").gsub(/_{2,}/, "_") + storage_manager.file_url(self, :large) end def preview_file_url - if !has_preview? - return "/images/download-preview.png" - end - - "/data/preview/#{file_path_prefix}#{md5}.jpg" + storage_manager.file_url(self, :preview) end def complete_preview_file_url @@ -211,17 +199,9 @@ class Post < ApplicationRecord def open_graph_image_url if is_image? if has_large? - if Danbooru.config.build_large_file_url(self) =~ /http/ - large_file_url - else - "http://#{Danbooru.config.hostname}#{large_file_url}" - end + large_file_url else - if Danbooru.config.build_file_url(self) =~ /http/ - file_url - else - "http://#{Danbooru.config.hostname}#{file_url}" - end + file_url end else complete_preview_file_url diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index a1bf74ff6..1a59e50ce 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -257,13 +257,6 @@ module Danbooru # end end - def build_file_url(post) - "/data/#{post.file_path_prefix}/#{post.md5}.#{post.file_ext}" - end - - def build_large_file_url(post) - "/data/sample/#{post.file_path_prefix}#{Danbooru.config.large_image_prefix}#{post.md5}.#{post.large_file_ext}" - end #TAG CONFIGURATION From f0bf1bc66e46f5daa144ee15cf2c72410e4ce980 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 18 Mar 2018 16:33:26 -0500 Subject: [PATCH 06/10] posts: use storage manager to backup files. * Perform backups synchronously inside `distribute_files` instead of asynchronously in `queue_backup`. Asynchronous backups assumed that files are stored on the local filesystem, which isn't true in general. * Remove obsolete backup service classes. --- app/logical/backup_service.rb | 9 ------ app/logical/null_backup_service.rb | 9 ------ app/logical/s3_backup_service.rb | 52 ------------------------------ app/models/post.rb | 27 +++++----------- app/models/post_replacement.rb | 2 -- config/danbooru_default_config.rb | 32 ++++++++++-------- test/test_helper.rb | 1 + 7 files changed, 27 insertions(+), 105 deletions(-) delete mode 100644 app/logical/backup_service.rb delete mode 100644 app/logical/null_backup_service.rb delete mode 100644 app/logical/s3_backup_service.rb diff --git a/app/logical/backup_service.rb b/app/logical/backup_service.rb deleted file mode 100644 index c34cdc339..000000000 --- a/app/logical/backup_service.rb +++ /dev/null @@ -1,9 +0,0 @@ -class BackupService - def backup(file_path, options = {}) - raise NotImplementedError.new("#{self.class}.backup not implemented") - end - - def delete(file_path, options = {}) - raise NotImplementedError.new("#{self.class}.delete not implemented") - end -end diff --git a/app/logical/null_backup_service.rb b/app/logical/null_backup_service.rb deleted file mode 100644 index 3c15b0f10..000000000 --- a/app/logical/null_backup_service.rb +++ /dev/null @@ -1,9 +0,0 @@ -class NullBackupService - def backup(file_path, options = {}) - # do nothing - end - - def delete(file_path, options = {}) - # do nothing - end -end diff --git a/app/logical/s3_backup_service.rb b/app/logical/s3_backup_service.rb deleted file mode 100644 index 655ea9914..000000000 --- a/app/logical/s3_backup_service.rb +++ /dev/null @@ -1,52 +0,0 @@ -class S3BackupService < BackupService - attr_reader :client, :bucket - - def initialize(client: nil, bucket: Danbooru.config.aws_s3_bucket_name) - @credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key) - @client = client || Aws::S3::Client.new(credentials: @credentials, region: "us-east-1", logger: Logger.new(STDOUT)) - @bucket = bucket - end - - def backup(file_path, type: nil, **options) - keys = s3_keys(file_path, type) - keys.each do |key| - upload_to_s3(key, file_path) - end - end - - def delete(file_path, type: nil) - keys = s3_keys(file_path, type) - keys.each do |key| - delete_from_s3(key) - end - end - -protected - def s3_keys(file_path, type) - name = File.basename(file_path) - - case type - when :original - [name] - when :preview - ["preview/#{name}"] - when :large - ["sample/#{name}"] - else - raise ArgumentError.new("Unknown type: #{type}") - end - end - - def delete_from_s3(key) - client.delete_object(bucket: bucket, key: key) - rescue Aws::S3::Errors::NoSuchKey - # ignore - end - - def upload_to_s3(key, file_path) - File.open(file_path, "rb") do |body| - base64_md5 = Digest::MD5.base64digest(File.read(file_path)) - client.put_object(acl: "public-read", bucket: bucket, key: key, body: body, content_md5: base64_md5) - end - end -end diff --git a/app/models/post.rb b/app/models/post.rb index 68eabfc55..43d8c6f00 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -29,7 +29,6 @@ class Post < ApplicationRecord before_save :set_tag_counts before_save :set_pool_category_pseudo_tags before_create :autoban - after_save :queue_backup, if: :md5_changed? after_save :create_version after_save :update_parent_on_save after_save :apply_post_metatags @@ -134,6 +133,10 @@ class Post < ApplicationRecord storage_manager.store_file(file, self, :original) storage_manager.store_file(sample_file, self, :large) if sample_file.present? storage_manager.store_file(preview_file, self, :preview) if preview_file.present? + + backup_storage_manager.store_file(file, self, :original) + backup_storage_manager.store_file(sample_file, self, :large) if sample_file.present? + backup_storage_manager.store_file(preview_file, self, :preview) if preview_file.present? end def file_path_prefix @@ -168,6 +171,10 @@ class Post < ApplicationRecord "#{file_path_prefix}#{md5}.#{file_ext}" end + def backup_storage_manager + Danbooru.config.backup_storage_manager + end + def storage_manager Danbooru.config.storage_manager end @@ -263,23 +270,6 @@ class Post < ApplicationRecord end end - module BackupMethods - extend ActiveSupport::Concern - - def queue_backup - Post.delay(queue: "default", priority: -1).backup_file(file_path, id: id, type: :original) - Post.delay(queue: "default", priority: -1).backup_file(large_file_path, id: id, type: :large) if has_large? - Post.delay(queue: "default", priority: -1).backup_file(preview_file_path, id: id, type: :preview) if has_preview? - end - - module ClassMethods - def backup_file(file_path, options = {}) - backup_service = Danbooru.config.backup_service - backup_service.backup(file_path, options) - end - end - end - module ImageMethods def twitter_card_supported? image_width.to_i >= 280 && image_height.to_i >= 150 @@ -1800,7 +1790,6 @@ class Post < ApplicationRecord end include FileMethods - include BackupMethods include ImageMethods include ApprovalMethods include PresenterMethods diff --git a/app/models/post_replacement.rb b/app/models/post_replacement.rb index b6b3fd11b..b8dd69fed 100644 --- a/app/models/post_replacement.rb +++ b/app/models/post_replacement.rb @@ -71,8 +71,6 @@ class PostReplacement < ApplicationRecord if md5_changed post.comments.create!({creator: User.system, body: comment_replacement_message, do_not_bump_post: true}, without_protection: true) - else - post.queue_backup end save! diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 1a59e50ce..ecbb9b63f 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -99,20 +99,6 @@ module Danbooru true end - # What method to use to backup images. - # - # NullBackupService: Don't backup images at all. - # - # S3BackupService: Backup to Amazon S3. Must configure aws_access_key_id, - # aws_secret_access_key, and aws_s3_bucket_name. Bucket must exist and be writable. - def backup_service - if Rails.env.production? - S3BackupService.new - else - NullBackupService.new - end - end - # What method to use to store images. # local_flat: Store every image in one directory. # local_hierarchy: Store every image in a hierarchical directory, based on the post's MD5 hash. On some file systems this may be faster. @@ -257,6 +243,24 @@ module Danbooru # end end + # The method to use for backing up image files. + def backup_storage_manager + # Don't perform any backups. + StorageManager::Null.new + + # Backup files to /mnt/backup on the local filesystem. + # StorageManager::Local.new(base_dir: "/mnt/backup", hierarchical: false) + + # Backup files to /mnt/backup on a remote system. Configure SSH settings + # in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) + # StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {}) + + # Backup files to an S3 bucket. The bucket must already exist and be + # writable by you. Configure your S3 settings in aws_region and + # aws_credentials below, or in the s3_options param (ref: + # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method) + # StorageManager::S3.new("my_s3_bucket_name", s3_options: {}) + end #TAG CONFIGURATION diff --git a/test/test_helper.rb b/test/test_helper.rb index 19474ca49..477e9eac2 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -41,6 +41,7 @@ class ActiveSupport::TestCase storage_manager = StorageManager::Local.new(base_dir: "#{Rails.root}/public/data/test") Danbooru.config.stubs(:storage_manager).returns(storage_manager) + Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new) end teardown do From b7f7187f632f22e66adae6ffa0163b2b935c48ae Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 18 Mar 2018 17:57:09 -0500 Subject: [PATCH 07/10] posts: use storage manager to delete files. --- app/models/post.rb | 39 +++++++++++++--------------------- app/models/post_replacement.rb | 2 +- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/app/models/post.rb b/app/models/post.rb index 43d8c6f00..c6b3d178f 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -93,40 +93,31 @@ class Post < ApplicationRecord extend ActiveSupport::Concern module ClassMethods - def delete_files(post_id, file_path, large_file_path, preview_file_path, force: false) - unless force - # XXX should pass in the md5 instead of parsing it. - preview_file_path =~ %r!/data/preview/(?:test\.)?([a-z0-9]{32})\.jpg\z! - md5 = $1 - - if Post.where(md5: md5).exists? - raise DeletionError.new("Files still in use; skipping deletion.") - end + def delete_files(post_id, md5, file_ext, force: false) + if Post.where(md5: md5).exists? && !force + raise DeletionError.new("Files still in use; skipping deletion.") end - backup_service = Danbooru.config.backup_service - backup_service.delete(file_path, type: :original) - backup_service.delete(large_file_path, type: :large) - backup_service.delete(preview_file_path, type: :preview) + Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :original) + Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :large) + Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :preview) - # the large file and the preview don't necessarily exist. if so errors will be ignored. - FileUtils.rm_f(file_path) - FileUtils.rm_f(large_file_path) - FileUtils.rm_f(preview_file_path) - - RemoteFileManager.new(file_path).delete - RemoteFileManager.new(large_file_path).delete - RemoteFileManager.new(preview_file_path).delete + Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :original) + Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :large) + Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :preview) if Danbooru.config.cloudflare_key - md5, ext = File.basename(file_path).split(".") - CloudflareService.new.delete(md5, ext) + CloudflareService.new.delete(md5, file_ext) end end end + def queue_delete_files(grace_period) + Post.delay(queue: "default", run_at: Time.now + grace_period).delete_files(id, md5, file_ext) + end + def delete_files - Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true) + Post.delete_files(id, md5, file_ext, force: true) end def distribute_files(file, sample_file, preview_file) diff --git a/app/models/post_replacement.rb b/app/models/post_replacement.rb index b8dd69fed..e709942fb 100644 --- a/app/models/post_replacement.rb +++ b/app/models/post_replacement.rb @@ -49,7 +49,7 @@ class PostReplacement < ApplicationRecord # md5/file_ext to delete the old files. if saving the post fails, # this is rolled back so the job won't run. if md5_changed - Post.delay(queue: "default", run_at: Time.now + DELETION_GRACE_PERIOD).delete_files(post.id, post.file_path, post.large_file_path, post.preview_file_path) + post.queue_delete_files(DELETION_GRACE_PERIOD) end self.file_ext = upload.file_ext From 41a4ff15cd73420f1d87748e735d3ed9b0b64878 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 18 Mar 2018 18:11:44 -0500 Subject: [PATCH 08/10] posts: remove unused file path / url methods. * Remove file_path_for, cropped_file_url (dead code) * Remove complete_preview_file_url (preview_file_url now returns absolute links) * Remove `file_name` (only used for Download link in sidebar) --- app/controllers/iqdb_queries_controller.rb | 2 +- app/models/post.rb | 60 ++----------------- app/presenters/post_presenter.rb | 4 ++ app/views/comments/index.atom.builder | 2 +- .../posts/partials/show/_options.html.erb | 2 +- test/test_helpers/iqdb_test_helper.rb | 2 +- 6 files changed, 13 insertions(+), 59 deletions(-) diff --git a/app/controllers/iqdb_queries_controller.rb b/app/controllers/iqdb_queries_controller.rb index b9aae52c0..62a9b93a5 100644 --- a/app/controllers/iqdb_queries_controller.rb +++ b/app/controllers/iqdb_queries_controller.rb @@ -49,7 +49,7 @@ protected def create_by_post @post = Post.find(params[:post_id]) - @download = Iqdb::Download.new(@post.complete_preview_file_url) + @download = Iqdb::Download.new(@post.preview_file_url) @download.find_similar @results = @download.matches end diff --git a/app/models/post.rb b/app/models/post.rb index c6b3d178f..9e6a211e0 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -130,38 +130,6 @@ class Post < ApplicationRecord backup_storage_manager.store_file(preview_file, self, :preview) if preview_file.present? end - def file_path_prefix - Rails.env == "test" ? "test." : "" - end - - def file_path - "#{Rails.root}/public/data/#{file_path_prefix}#{md5}.#{file_ext}" - end - - def large_file_path - if has_large? - "#{Rails.root}/public/data/sample/#{file_path_prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}" - else - file_path - end - end - - def large_file_ext - if is_ugoira? - "webm" - else - "jpg" - end - end - - def preview_file_path - "#{Rails.root}/public/data/preview/#{file_path_prefix}#{md5}.jpg" - end - - def file_name - "#{file_path_prefix}#{md5}.#{file_ext}" - end - def backup_storage_manager Danbooru.config.backup_storage_manager end @@ -178,10 +146,6 @@ class Post < ApplicationRecord storage_manager.file_url(self, :original) end - # this is for the 640x320 version - def cropped_file_url - end - def large_file_url storage_manager.file_url(self, :large) end @@ -190,10 +154,6 @@ class Post < ApplicationRecord storage_manager.file_url(self, :preview) end - def complete_preview_file_url - "http://#{Danbooru.config.hostname}#{preview_file_url}" - end - def open_graph_image_url if is_image? if has_large? @@ -202,7 +162,7 @@ class Post < ApplicationRecord file_url end else - complete_preview_file_url + preview_file_url end end @@ -214,14 +174,6 @@ class Post < ApplicationRecord end end - def file_path_for(user) - if user.default_image_size == "large" && image_width > Danbooru.config.large_image_width - large_file_path - else - file_path - end - end - def is_image? file_ext =~ /jpg|jpeg|gif|png/i end @@ -247,9 +199,7 @@ class Post < ApplicationRecord end def has_preview? - # for video/ugoira we don't want to try and render a preview that - # might doesn't exist yet - is_image? || ((is_video? || is_ugoira?) && File.exists?(preview_file_path)) + is_image? || is_video? || is_ugoira? end def has_dimensions? @@ -257,7 +207,7 @@ class Post < ApplicationRecord end def has_ugoira_webm? - created_at < 1.minute.ago || (File.exists?(preview_file_path) && File.size(preview_file_path) > 0) + true end end @@ -1674,8 +1624,8 @@ class Post < ApplicationRecord end def update_iqdb_async - if File.exists?(preview_file_path) && Post.iqdb_enabled? - Post.iqdb_sqs_service.send_message("update\n#{id}\n#{complete_preview_file_url}") + if Post.iqdb_enabled? + Post.iqdb_sqs_service.send_message("update\n#{id}\n#{preview_file_url}") end end diff --git a/app/presenters/post_presenter.rb b/app/presenters/post_presenter.rb index 4fe901fbb..e0c70520a 100644 --- a/app/presenters/post_presenter.rb +++ b/app/presenters/post_presenter.rb @@ -135,6 +135,10 @@ class PostPresenter < Presenter @post.humanized_essential_tag_string end + def filename_for_download + "#{humanized_essential_tag_string} - #{@post.md5}.#{@post.file_ext}" + end + def categorized_tag_groups string = [] diff --git a/app/views/comments/index.atom.builder b/app/views/comments/index.atom.builder index 09d017868..19e1c8b75 100644 --- a/app/views/comments/index.atom.builder +++ b/app/views/comments/index.atom.builder @@ -11,7 +11,7 @@ atom_feed(root_url: comments_url(host: Danbooru.config.hostname)) do |feed| feed.entry(comment, published: comment.created_at, updated: comment.updated_at) do |entry| entry.title("@#{comment.creator_name} on post ##{comment.post_id} (#{comment.post.humanized_essential_tag_string})") entry.content(<<-EOS.strip_heredoc, type: "html") - + #{format_text(comment.body)} EOS diff --git a/app/views/posts/partials/show/_options.html.erb b/app/views/posts/partials/show/_options.html.erb index 81350deaa..79d83588d 100644 --- a/app/views/posts/partials/show/_options.html.erb +++ b/app/views/posts/partials/show/_options.html.erb @@ -4,7 +4,7 @@ <% if CurrentUser.is_member? %>
  • <%= link_to "Favorite", favorites_path(:post_id => post.id), :remote => true, :method => :post, :id => "add-to-favorites", :title => "Shortcut is F" %>
  • <%= link_to "Unfavorite", favorite_path(post), :remote => true, :method => :delete, :id => "remove-from-favorites" %>
    • -
  • <%= link_to_if post.visible?, "Download", post.file_url, :download => post.presenter.humanized_essential_tag_string + " - " + post.file_name %>
    • +
  • <%= link_to_if post.visible?, "Download", post.file_url, download: post.presenter.filename_for_download %>
  • <%= link_to "Add to pool", "#", :id => "pool" %>
    • <% if post.is_note_locked? %>
  • Note locked
    • diff --git a/test/test_helpers/iqdb_test_helper.rb b/test/test_helpers/iqdb_test_helper.rb index 9bcdec4f4..edd941f6e 100644 --- a/test/test_helpers/iqdb_test_helper.rb +++ b/test/test_helpers/iqdb_test_helper.rb @@ -23,7 +23,7 @@ module IqdbTestHelper end def mock_iqdb_matches!(post_or_source, matches) - source = post_or_source.is_a?(Post) ? post_or_source.complete_preview_file_url : post_or_source + source = post_or_source.is_a?(Post) ? post_or_source.preview_file_url : post_or_source url = "http://localhost:3004/similar?key=hunter2&url=#{CGI.escape source}&ref" body = matches.map { |post| { post_id: post.id } }.to_json From d089be9f8ae3a89c4791531363b71bc0c1ab153f Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 19 Mar 2018 23:17:24 -0500 Subject: [PATCH 09/10] tests: fix upload tests. --- test/factories/upload.rb | 47 +---- .../post_replacements_controller_test.rb | 2 +- test/test_helper.rb | 1 + test/test_helpers/download_helper.rb | 12 +- test/test_helpers/upload_test_helper.rb | 25 +-- test/unit/downloads/art_station_test.rb | 10 +- test/unit/downloads/deviant_art_test.rb | 5 +- test/unit/downloads/file_test.rb | 18 +- test/unit/downloads/pixiv_test.rb | 10 +- test/unit/pixiv_ugoira_converter_test.rb | 19 +- test/unit/post_replacement_test.rb | 43 ++--- test/unit/post_test.rb | 10 +- test/unit/upload_test.rb | 167 +++++------------- 13 files changed, 96 insertions(+), 273 deletions(-) diff --git a/test/factories/upload.rb b/test/factories/upload.rb index c4ac00104..cd6a89d0d 100644 --- a/test/factories/upload.rb +++ b/test/factories/upload.rb @@ -1,5 +1,3 @@ -require 'fileutils' - FactoryGirl.define do factory(:upload) do rating "s" @@ -15,51 +13,10 @@ FactoryGirl.define do end factory(:jpg_upload) do - content_type "image/jpeg" file do f = Tempfile.new - f.write(File.read("#{Rails.root}/test/files/test.jpg")) - f.seek(0) - f - end - end - - factory(:exif_jpg_upload) do - content_type "image/jpeg" - file_path do - FileUtils.cp("#{Rails.root}/test/files/test-exif-small.jpg", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test-exif-small.jpg" - end - end - - factory(:blank_jpg_upload) do - content_type "image/jpeg" - file_path do - FileUtils.cp("#{Rails.root}/test/files/test-blank.jpg", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test-blank.jpg" - end - end - - factory(:large_jpg_upload) do - file_ext "jpg" - content_type "image/jpeg" - file_path do - FileUtils.cp("#{Rails.root}/test/files/test-large.jpg", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test-large.jpg" - end - end - - factory(:png_upload) do - file_path do - FileUtils.cp("#{Rails.root}/test/files/test.png", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test.png" - end - end - - factory(:gif_upload) do - file_path do - FileUtils.cp("#{Rails.root}/test/files/test.gif", "#{Rails.root}/tmp") - "#{Rails.root}/tmp/test.gif" + IO.copy_stream("#{Rails.root}/test/files/test.jpg", f.path) + ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "test.jpg") end end end diff --git a/test/functional/post_replacements_controller_test.rb b/test/functional/post_replacements_controller_test.rb index c4d05597e..5944215de 100644 --- a/test/functional/post_replacements_controller_test.rb +++ b/test/functional/post_replacements_controller_test.rb @@ -39,7 +39,7 @@ class PostReplacementsControllerTest < ActionController::TestCase assert_response :success assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source) assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5) - assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file(:original)).hexdigest) end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 477e9eac2..9a7dd29da 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -45,6 +45,7 @@ class ActiveSupport::TestCase end teardown do + FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir) Cache.clear end end diff --git a/test/test_helpers/download_helper.rb b/test/test_helpers/download_helper.rb index 9097e7cea..8381f4629 100644 --- a/test/test_helpers/download_helper.rb +++ b/test/test_helpers/download_helper.rb @@ -1,18 +1,14 @@ module DownloadTestHelper def assert_downloaded(expected_filesize, source) - tempfile = Tempfile.new("danbooru-test") - download = Downloads::File.new(source, tempfile.path) - assert_nothing_raised(Downloads::File::Error) do - download.download! + tempfile = Downloads::File.new(source).download! + assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}") + tempfile.close! end - - assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}") end def assert_rewritten(expected_source, test_source) - tempfile = Tempfile.new("danbooru-test") - download = Downloads::File.new(test_source, tempfile.path) + download = Downloads::File.new(test_source) rewritten_source, _, _ = download.before_download(test_source, {}) assert_match(expected_source, rewritten_source, "Tested source URL: #{test_source}") diff --git a/test/test_helpers/upload_test_helper.rb b/test/test_helpers/upload_test_helper.rb index 3b29574e6..66b404278 100644 --- a/test/test_helpers/upload_test_helper.rb +++ b/test/test_helpers/upload_test_helper.rb @@ -1,23 +1,10 @@ module UploadTestHelper - def upload_file(path, content_type, filename) - tempfile = Tempfile.new(filename) - FileUtils.copy_file(path, tempfile.path) + def upload_file(path) + file = Tempfile.new(binmode: true) + IO.copy_stream("#{Rails.root}/#{path}", file.path) + uploaded_file = ActionDispatch::Http::UploadedFile.new(tempfile: file, filename: File.basename(path)) - (class << tempfile; self; end).class_eval do - alias local_path path - define_method(:tempfile) {self} - define_method(:original_filename) {filename} - define_method(:content_type) {content_type} - end - - tempfile - end - - def upload_jpeg(path) - upload_file(path, "image/jpeg", File.basename(path)) - end - - def upload_zip(path) - upload_file(path, "application/zip", File.basename(path)) + yield uploaded_file if block_given? + uploaded_file end end diff --git a/test/unit/downloads/art_station_test.rb b/test/unit/downloads/art_station_test.rb index 077007b8d..528210bdb 100644 --- a/test/unit/downloads/art_station_test.rb +++ b/test/unit/downloads/art_station_test.rb @@ -5,9 +5,7 @@ module Downloads context "a download for a (small) artstation image" do setup do @source = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - @download.download! + @download = Downloads::File.new(@source) end should "download the large image instead" do @@ -18,8 +16,7 @@ module Downloads context "for an image where an original does not exist" do setup do @source = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) + @download = Downloads::File.new(@source) @download.download! end @@ -38,8 +35,7 @@ module Downloads context "a download for a https://$artist.artstation.com/projects/$id page" do setup do @source = "https://dantewontdie.artstation.com/projects/YZK5q" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) + @download = Downloads::File.new(@source) @download.download! end diff --git a/test/unit/downloads/deviant_art_test.rb b/test/unit/downloads/deviant_art_test.rb index 42013174d..7cb58c5bd 100644 --- a/test/unit/downloads/deviant_art_test.rb +++ b/test/unit/downloads/deviant_art_test.rb @@ -5,9 +5,8 @@ module Downloads context "a download for a deviant art html page" do setup do @source = "http://starbitt.deviantart.com/art/09271X-636962118" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - @download.download! + @download = Downloads::File.new(@source) + @tempfile = @download.download! end should "set the html page as the source" do diff --git a/test/unit/downloads/file_test.rb b/test/unit/downloads/file_test.rb index 126b4f174..7698941d8 100644 --- a/test/unit/downloads/file_test.rb +++ b/test/unit/downloads/file_test.rb @@ -5,12 +5,7 @@ module Downloads context "A twitter video download" do setup do @source = "https://twitter.com/CincinnatiZoo/status/859073537713328129" - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - end - - teardown do - @tempfile.close + @download = Downloads::File.new(@source) end should "preserve the twitter source" do @@ -22,12 +17,8 @@ module Downloads context "A post download" do setup do @source = "http://www.google.com/intl/en_ALL/images/logo.gif" + @download = Downloads::File.new(@source) @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new(@source, @tempfile.path) - end - - teardown do - @tempfile.close end context "that fails" do @@ -49,10 +40,9 @@ module Downloads end should "store the file in the tempfile path" do - @download.download! + tempfile = @download.download! assert_equal(@source, @download.source) - assert(::File.exists?(@tempfile.path), "temp file should exist") - assert(::File.size(@tempfile.path) > 0, "should have data") + assert_operator(tempfile.size, :>, 0, "should have data") end end end diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb index 66cabf113..3441ee81b 100644 --- a/test/unit/downloads/pixiv_test.rb +++ b/test/unit/downloads/pixiv_test.rb @@ -4,13 +4,9 @@ module Downloads class PixivTest < ActiveSupport::TestCase context "An ugoira site for pixiv" do setup do - @tempfile = Tempfile.new("danbooru-test") - @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364", @tempfile.path) - @download.download! - end - - teardown do - @tempfile.unlink + @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @tempfile = @download.download! + @tempfile.close! end should "capture the data" do diff --git a/test/unit/pixiv_ugoira_converter_test.rb b/test/unit/pixiv_ugoira_converter_test.rb index ed7b9a146..bba9e4c58 100644 --- a/test/unit/pixiv_ugoira_converter_test.rb +++ b/test/unit/pixiv_ugoira_converter_test.rb @@ -3,9 +3,7 @@ require "test_helper" class PixivUgoiraConverterTest < ActiveSupport::TestCase context "An ugoira converter" do setup do - @zipped_body = "#{Rails.root}/test/fixtures/ugoira.zip" - @write_file = Tempfile.new("converted") - @preview_write_file = Tempfile.new("preview") + @zipfile = upload_file("test/fixtures/ugoira.zip").tempfile @frame_data = [ {"file" => "000000.jpg", "delay" => 200}, {"file" => "000001.jpg", "delay" => 200}, @@ -15,16 +13,11 @@ class PixivUgoiraConverterTest < ActiveSupport::TestCase ] end - teardown do - @write_file.unlink - @preview_write_file.unlink - end - should "output to webm" do - @converter = PixivUgoiraConverter - @converter.convert(@zipped_body, @write_file.path, @preview_write_file.path, @frame_data) - assert_operator(File.size(@write_file.path), :>, 1_000) - assert_operator(File.size(@preview_write_file.path), :>, 0) + sample_file = PixivUgoiraConverter.generate_webm(@zipfile, @frame_data) + preview_file = PixivUgoiraConverter.generate_preview(@zipfile) + assert_operator(sample_file.size, :>, 1_000) + assert_operator(preview_file.size, :>, 0) end end -end \ No newline at end of file +end diff --git a/test/unit/post_replacement_test.rb b/test/unit/post_replacement_test.rb index 08ecea2e5..8d8df7653 100644 --- a/test/unit/post_replacement_test.rb +++ b/test/unit/post_replacement_test.rb @@ -1,15 +1,6 @@ require 'test_helper' class PostReplacementTest < ActiveSupport::TestCase - def upload_file(path, filename, &block) - Tempfile.open do |file| - file.write(File.read(path)) - file.seek(0) - uploaded_file = ActionDispatch::Http::UploadedFile.new(tempfile: file, filename: filename) - yield uploaded_file - end - end - def setup super @@ -68,7 +59,7 @@ class PostReplacementTest < ActiveSupport::TestCase assert_equal(5969, @post.file_size) assert_equal("png", @post.file_ext) assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", @post.md5) - assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file).hexdigest) assert_equal("https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png", @post.source) end end @@ -102,7 +93,7 @@ class PostReplacementTest < ActiveSupport::TestCase assert_equal(8558, @post.file_size) assert_equal("gif", @post.file_ext) assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5) - assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file).hexdigest) assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source) end @@ -155,18 +146,17 @@ class PostReplacementTest < ActiveSupport::TestCase assert_equal(16275, @post.file_size) assert_equal("png", @post.file_ext) assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5) - assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest) assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source) assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url) end - should "delete the old files after three days" do - old_file_path, old_preview_file_path, old_large_file_path = @post.file_path, @post.preview_file_path, @post.large_file_path + should "delete the old files after thirty days" do + old_file_path, old_preview_file_path = @post.file(:original).path, @post.file(:preview).path @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") assert(File.exists?(old_file_path)) assert(File.exists?(old_preview_file_path)) - assert(File.exists?(old_large_file_path)) Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do Delayed::Worker.new.work_off @@ -174,7 +164,6 @@ class PostReplacementTest < ActiveSupport::TestCase assert_not(File.exists?(old_file_path)) assert_not(File.exists?(old_preview_file_path)) - assert_not(File.exists?(old_large_file_path)) end end @@ -188,7 +177,7 @@ class PostReplacementTest < ActiveSupport::TestCase assert_equal(2804, @post.file_size) assert_equal("zip", @post.file_ext) assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5) - assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file_path).hexdigest) + assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest) assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source) assert_equal([{"file"=>"000000.jpg", "delay"=>125}, {"file"=>"000001.jpg", "delay"=>125}], @post.pixiv_ugoira_frame_data.data) @@ -201,17 +190,15 @@ class PostReplacementTest < ActiveSupport::TestCase @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - assert(File.exists?(@post.file_path)) - assert(File.exists?(@post.preview_file_path)) - assert(File.exists?(@post.large_file_path)) + assert_nothing_raised { @post.file(:original) } + assert_nothing_raised { @post.file(:preview) } Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do Delayed::Worker.new.work_off end - assert(File.exists?(@post.file_path)) - assert(File.exists?(@post.preview_file_path)) - assert(File.exists?(@post.large_file_path)) + assert_nothing_raised { @post.file(:original) } + assert_nothing_raised { @post.file(:preview) } end end @@ -231,14 +218,14 @@ class PostReplacementTest < ActiveSupport::TestCase Delayed::Worker.new.work_off end - assert(File.exists?(@post1.file_path)) - assert(File.exists?(@post2.file_path)) + assert_nothing_raised { @post1.file(:original) } + assert_nothing_raised { @post2.file(:original) } end end context "a post with an uploaded file" do should "work" do - upload_file("#{Rails.root}/test/files/test.png", "test.png") do |file| + upload_file("test/files/test.png") do |file| @post.replace!(replacement_file: file, replacement_url: "") assert_equal(@post.md5, Digest::MD5.file(file.tempfile).hexdigest) assert_equal("file://test.png", @post.replacements.last.replacement_url) @@ -268,7 +255,7 @@ class PostReplacementTest < ActiveSupport::TestCase context "a post with the same file" do should "not raise a duplicate error" do - upload_file("#{Rails.root}/test/files/test.jpg", "test.jpg") do |file| + upload_file("test/files/test.jpg") do |file| assert_nothing_raised do @post.replace!(replacement_file: file, replacement_url: "") end @@ -276,7 +263,7 @@ class PostReplacementTest < ActiveSupport::TestCase end should "not queue a deletion or log a comment" do - upload_file("#{Rails.root}/test/files/test.jpg", "test.jpg") do |file| + upload_file("test/files/test.jpg") do |file| assert_no_difference(["@post.comments.count"]) do @post.replace!(replacement_file: file, replacement_url: "") end diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index d805c11fe..a09ba0134 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -35,17 +35,15 @@ class PostTest < ActiveSupport::TestCase end should "delete the files" do - assert_equal(true, File.exists?(@post.preview_file_path)) - assert_equal(true, File.exists?(@post.large_file_path)) - assert_equal(true, File.exists?(@post.file_path)) + assert_nothing_raised { @post.file(:preview) } + assert_nothing_raised { @post.file(:original) } TestAfterCommit.with_commits(true) do @post.expunge! end - assert_equal(false, File.exists?(@post.preview_file_path)) - assert_equal(false, File.exists?(@post.large_file_path)) - assert_equal(false, File.exists?(@post.file_path)) + assert_raise(StandardError) { @post.file(:preview) } + assert_raise(StandardError) { @post.file(:original) } end should "remove all favorites" do diff --git a/test/unit/upload_test.rb b/test/unit/upload_test.rb index 022d66e6a..2a4dc196d 100644 --- a/test/unit/upload_test.rb +++ b/test/unit/upload_test.rb @@ -17,21 +17,15 @@ class UploadTest < ActiveSupport::TestCase teardown do CurrentUser.user = nil CurrentUser.ip_addr = nil - - @upload.delete_temp_file if @upload end context "An upload" do - teardown do - FileUtils.rm_f(Dir.glob("#{Rails.root}/tmp/test.*")) - end - context "from a user that is limited" do setup do CurrentUser.user = FactoryGirl.create(:user, :created_at => 1.year.ago) User.any_instance.stubs(:upload_limit).returns(0) end - + should "fail creation" do @upload = FactoryGirl.build(:jpg_upload, :tag_string => "") @upload.save @@ -41,73 +35,51 @@ class UploadTest < ActiveSupport::TestCase context "image size calculator" do should "discover the dimensions for a compressed SWF" do - @upload = FactoryGirl.create(:upload, :file_path => "#{Rails.root}/test/files/compressed.swf") - @upload.calculate_dimensions(@upload.file_path) - assert_equal(607, @upload.image_width) - assert_equal(756, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/compressed.swf")) + assert_equal([607, 756], @upload.calculate_dimensions) end should "discover the dimensions for a JPG with JFIF data" do @upload = FactoryGirl.create(:jpg_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(500, @upload.image_width) - assert_equal(335, @upload.image_height) + assert_equal([500, 335], @upload.calculate_dimensions) end should "discover the dimensions for a JPG with EXIF data" do - @upload = FactoryGirl.create(:exif_jpg_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(529, @upload.image_width) - assert_equal(600, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test-exif-small.jpg")) + assert_equal([529, 600], @upload.calculate_dimensions) end should "discover the dimensions for a JPG with no header data" do - @upload = FactoryGirl.create(:blank_jpg_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(668, @upload.image_width) - assert_equal(996, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test-blank.jpg")) + assert_equal([668, 996], @upload.calculate_dimensions) end should "discover the dimensions for a PNG" do - @upload = FactoryGirl.create(:png_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(768, @upload.image_width) - assert_equal(1024, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.png")) + assert_equal([768, 1024], @upload.calculate_dimensions) end should "discover the dimensions for a GIF" do - @upload = FactoryGirl.create(:gif_upload) - assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} - assert_equal(400, @upload.image_width) - assert_equal(400, @upload.image_height) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.gif")) + assert_equal([400, 400], @upload.calculate_dimensions) end end context "content type calculator" do should "know how to parse jpeg, png, gif, and swf file headers" do - @upload = FactoryGirl.create(:jpg_upload) - assert_equal("image/jpeg", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.jpg")) - assert_equal("image/gif", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.gif")) - assert_equal("image/png", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.png")) - assert_equal("application/x-shockwave-flash", @upload.file_header_to_content_type("#{Rails.root}/test/files/compressed.swf")) - assert_equal("application/octet-stream", @upload.file_header_to_content_type("#{Rails.root}/README.md")) - end - - should "know how to parse jpeg, png, gif, and swf content types" do - @upload = FactoryGirl.create(:jpg_upload) - assert_equal("jpg", @upload.content_type_to_file_ext("image/jpeg")) - assert_equal("gif", @upload.content_type_to_file_ext("image/gif")) - assert_equal("png", @upload.content_type_to_file_ext("image/png")) - assert_equal("swf", @upload.content_type_to_file_ext("application/x-shockwave-flash")) - assert_equal("bin", @upload.content_type_to_file_ext("")) + @upload = FactoryGirl.build(:jpg_upload) + assert_equal("jpg", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.jpg"))) + assert_equal("gif", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.gif"))) + assert_equal("png", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.png"))) + assert_equal("swf", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/compressed.swf"))) + assert_equal("bin", @upload.file_header_to_file_ext(File.open("#{Rails.root}/README.md"))) end end context "downloader" do context "for a zip that is not an ugoira" do should "not validate" do - FileUtils.cp("#{Rails.root}/test/files/invalid_ugoira.zip", "#{Rails.root}/tmp") - @upload = Upload.create(:file => upload_zip("#{Rails.root}/tmp/invalid_ugoira.zip"), :rating => "q", :tag_string => "xxx") + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/invalid_ugoira.zip")) @upload.process! assert_equal("error: RuntimeError - missing frame data for ugoira", @upload.status) end @@ -116,30 +88,15 @@ class UploadTest < ActiveSupport::TestCase context "that is a pixiv ugoira" do setup do @url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654" - @upload = FactoryGirl.create(:source_upload, :source => @url, :tag_string => "ugoira") - @output_file = Tempfile.new("download") + @upload = FactoryGirl.create(:upload, :source => @url, :tag_string => "ugoira") end - teardown do - @output_file.unlink - end - should "process successfully" do - @upload.download_from_source(@output_file.path) - assert_operator(File.size(@output_file.path), :>, 1_000) - assert_equal("application/zip", @upload.file_header_to_content_type(@output_file.path)) - assert_equal("zip", @upload.content_type_to_file_ext(@upload.file_header_to_content_type(@output_file.path))) + _, _, output_file = @upload.download_from_source(@url, "") + assert_operator(output_file.size, :>, 1_000) + assert_equal("zip", @upload.file_header_to_file_ext(output_file)) end end - - should "initialize the final path after downloading a file" do - @upload = FactoryGirl.create(:source_upload) - path = "#{Rails.root}/tmp/test.download.jpg" - assert_nothing_raised {@upload.download_from_source(path)} - assert(File.exists?(path)) - assert_equal(8558, File.size(path)) - assert_equal(path, @upload.file_path) - end end context "determining if a file is downloadable" do @@ -161,46 +118,32 @@ class UploadTest < ActiveSupport::TestCase context "file processor" do should "parse and process a cgi file representation" do - FileUtils.cp("#{Rails.root}/test/files/test.jpg", "#{Rails.root}/tmp") - @upload = Upload.new(:file => upload_jpeg("#{Rails.root}/tmp/test.jpg")) - assert_nothing_raised {@upload.convert_cgi_file} - assert(File.exists?(@upload.file_path)) - assert_equal(28086, File.size(@upload.file_path)) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.jpg")) + assert_nothing_raised {@upload.process_upload} + assert_equal(28086, @upload.file_size) end should "process a transparent png" do - FileUtils.cp("#{Rails.root}/test/files/alpha.png", "#{Rails.root}/tmp") - @upload = Upload.new(:file => upload_file("#{Rails.root}/tmp/alpha.png", "image/png", "alpha.png")) - assert_nothing_raised {@upload.convert_cgi_file} - assert(File.exists?(@upload.file_path)) - assert_equal(1136, File.size(@upload.file_path)) + @upload = FactoryGirl.create(:upload, file: upload_file("test/files/alpha.png")) + assert_nothing_raised {@upload.process_upload} + assert_equal(1136, @upload.file_size) end end context "hash calculator" do should "caculate the hash" do @upload = FactoryGirl.create(:jpg_upload) - @upload.calculate_hash(@upload.file_path) + @upload.process_upload assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5) end end context "resizer" do - teardown do - FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/preview/test.*.jpg")) - FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/sample/test.*.jpg")) - FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/test.*.jpg")) - end - should "generate several resized versions of the image" do - @upload = FactoryGirl.create(:large_jpg_upload) - @upload.calculate_hash(@upload.file_path) - @upload.calculate_dimensions(@upload.file_path) - assert_nothing_raised {@upload.generate_resizes(@upload.file_path)} - assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.small_image_width))) - assert(File.size(@upload.resized_file_path_for(Danbooru.config.small_image_width)) > 0) - assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.large_image_width))) - assert(File.size(@upload.resized_file_path_for(Danbooru.config.large_image_width)) > 0) + @upload = FactoryGirl.create(:upload, file_ext: "jpg", image_width: 1356, image_height: 911, file: upload_file("test/files/test-large.jpg")) + preview_file, sample_file = @upload.generate_resizes + assert_operator(preview_file.size, :>, 1_000) + assert_operator(sample_file.size, :>, 1_000) end end @@ -215,13 +158,10 @@ class UploadTest < ActiveSupport::TestCase context "with an artist commentary" do setup do @upload = FactoryGirl.create(:source_upload, - :rating => "s", - :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" - ) - @upload.include_artist_commentary = "1" - @upload.artist_commentary_title = "" - @upload.artist_commentary_desc = "blah" + include_artist_commentary: "1", + artist_commentary_title: "", + artist_commentary_desc: "blah", + ) end should "create an artist commentary when processed" do @@ -258,12 +198,7 @@ class UploadTest < ActiveSupport::TestCase end should "process completely for a pixiv ugoira" do - @upload = FactoryGirl.create(:source_upload, - :source => "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654", - :rating => "s", - :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" - ) + @upload = FactoryGirl.create(:source_upload, source: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654") assert_difference(["PixivUgoiraFrameData.count", "Post.count"]) do @upload.process! assert_equal([], @upload.errors.full_messages) @@ -274,18 +209,17 @@ class UploadTest < ActiveSupport::TestCase assert_equal(60, post.image_width) assert_equal(60, post.image_height) assert_equal("https://i.pximg.net/img-zip-ugoira/img/2014/10/05/23/42/23/46378654_ugoira1920x1080.zip", post.source) - assert_operator(File.size(post.large_file_path), :>, 0) - assert_operator(File.size(post.preview_file_path), :>, 0) + assert_nothing_raised { post.file(:original) } + assert_nothing_raised { post.file(:preview) } end should "process completely for an uploaded image" do @upload = FactoryGirl.create(:jpg_upload, :rating => "s", :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" + :tag_string => "hoge foo", + :file => upload_file("test/files/test.jpg"), ) - @upload.file = upload_jpeg("#{Rails.root}/test/files/test.jpg") - @upload.convert_cgi_file assert_difference("Post.count") do assert_nothing_raised {@upload.process!} @@ -297,8 +231,8 @@ class UploadTest < ActiveSupport::TestCase assert_equal("127.0.0.1", post.uploader_ip_addr.to_s) assert_equal(@upload.md5, post.md5) assert_equal("jpg", post.file_ext) - assert(File.exists?(post.file_path)) - assert_equal(28086, File.size(post.file_path)) + assert_nothing_raised { post.file(:original) } + assert_equal(28086, post.file(:original).size) assert_equal(post.id, @upload.post_id) assert_equal("completed", @upload.status) end @@ -310,16 +244,5 @@ class UploadTest < ActiveSupport::TestCase assert_nothing_raised {@upload.process!} end end - - should "delete the temporary file upon completion" do - @upload = FactoryGirl.create(:source_upload, - :rating => "s", - :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" - ) - - @upload.process! - assert(!File.exists?(@upload.temp_file_path)) - end end end From 619a2055fe40a3d21a398c8cd1ffec865ef8e46f Mon Sep 17 00:00:00 2001 From: evazion Date: Sat, 17 Mar 2018 16:04:09 -0500 Subject: [PATCH 10/10] tests: add storage manager tests. --- test/unit/storage_manager_test.rb | 124 ++++++++++++++++++++++++++++++ test/unit/upload_test.rb | 8 ++ 2 files changed, 132 insertions(+) create mode 100644 test/unit/storage_manager_test.rb diff --git a/test/unit/storage_manager_test.rb b/test/unit/storage_manager_test.rb new file mode 100644 index 000000000..5b4adc911 --- /dev/null +++ b/test/unit/storage_manager_test.rb @@ -0,0 +1,124 @@ +require 'test_helper' + +class StorageManagerTest < ActiveSupport::TestCase + BASE_DIR = "#{Rails.root}/tmp/test-storage" + + setup do + CurrentUser.ip_addr = "127.0.0.1" + end + + context "StorageManager::Local" do + setup do + @storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data") + end + + teardown do + FileUtils.rm_rf(BASE_DIR) + end + + context "#store method" do + should "store the file" do + @storage_manager.store(StringIO.new("data"), "#{BASE_DIR}/test.txt") + + assert("data", File.read("#{BASE_DIR}/test.txt")) + end + + should "overwrite the file if it already exists" do + @storage_manager.store(StringIO.new("foo"), "#{BASE_DIR}/test.txt") + @storage_manager.store(StringIO.new("bar"), "#{BASE_DIR}/test.txt") + + assert("bar", File.read("#{BASE_DIR}/test.txt")) + end + end + + context "#delete method" do + should "delete the file" do + @storage_manager.store(StringIO.new("data"), "test.txt") + @storage_manager.delete("test.txt") + + assert_not(File.exist?("#{BASE_DIR}/test.txt")) + end + + should "not fail if the file doesn't exist" do + assert_nothing_raised { @storage_manager.delete("dne.txt") } + end + end + + context "#store_file and #delete_file methods" do + setup do + @post = FactoryGirl.create(:post, file_ext: "png") + + @storage_manager.store_file(StringIO.new("data"), @post, :preview) + @storage_manager.store_file(StringIO.new("data"), @post, :large) + @storage_manager.store_file(StringIO.new("data"), @post, :original) + + @file_path = "#{BASE_DIR}/preview/#{@post.md5}.jpg" + @large_file_path = "#{BASE_DIR}/sample/sample-#{@post.md5}.jpg" + @preview_file_path = "#{BASE_DIR}/#{@post.md5}.#{@post.file_ext}" + end + + should "store the files at the correct path" do + assert(File.exist?(@file_path)) + assert(File.exist?(@large_file_path)) + assert(File.exist?(@preview_file_path)) + end + + should "delete the files" do + @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :preview) + @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :large) + @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :original) + + assert_not(File.exist?(@file_path)) + assert_not(File.exist?(@large_file_path)) + assert_not(File.exist?(@preview_file_path)) + end + end + + context "#file_url method" do + should "return the correct urls" do + @post = FactoryGirl.create(:post, file_ext: "png") + @storage_manager.stubs(:tagged_filenames).returns(false) + + assert_equal("/data/#{@post.md5}.png", @storage_manager.file_url(@post, :original)) + assert_equal("/data/sample/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large)) + assert_equal("/data/preview/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview)) + end + end + end + + context "StorageManager::Hybrid" do + setup do + @post1 = FactoryGirl.build(:post, id: 1, file_ext: "png") + @post2 = FactoryGirl.build(:post, id: 2, file_ext: "png") + + @storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type| + if id.odd? + StorageManager::Local.new(base_dir: "#{BASE_DIR}/i1", base_url: "/i1") + else + StorageManager::Local.new(base_dir: "#{BASE_DIR}/i2", base_url: "/i2") + end + end + end + + teardown do + FileUtils.rm_rf(BASE_DIR) + end + + context "#store_file method" do + should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do + @storage_manager.store_file(StringIO.new("post1"), @post1, :original) + @storage_manager.store_file(StringIO.new("post2"), @post2, :original) + + assert(File.exist?("#{BASE_DIR}/i1/#{@post1.md5}.png")) + assert(File.exist?("#{BASE_DIR}/i2/#{@post2.md5}.png")) + end + end + + context "#file_url method" do + should "generate /i1 urls for odd posts and /i2 urls for even posts" do + assert_equal("/i1/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original)) + assert_equal("/i2/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original)) + end + end + end +end diff --git a/test/unit/upload_test.rb b/test/unit/upload_test.rb index 2a4dc196d..cd7184c6f 100644 --- a/test/unit/upload_test.rb +++ b/test/unit/upload_test.rb @@ -195,6 +195,13 @@ class UploadTest < ActiveSupport::TestCase assert_equal(post.id, @upload.post_id) assert_equal("completed", @upload.status) end + + context "automatic tagging" do + should "tag animated png files" do + @upload = FactoryGirl.build(:upload, file_ext: "png", file: upload_file("test/files/apng/normal_apng.png")) + assert_equal("animated_png", @upload.automatic_tags) + end + end end should "process completely for a pixiv ugoira" do @@ -210,6 +217,7 @@ class UploadTest < ActiveSupport::TestCase assert_equal(60, post.image_height) assert_equal("https://i.pximg.net/img-zip-ugoira/img/2014/10/05/23/42/23/46378654_ugoira1920x1080.zip", post.source) assert_nothing_raised { post.file(:original) } + assert_nothing_raised { post.file(:large) } assert_nothing_raised { post.file(:preview) } end