Merge pull request #3577 from evazion/feat-storage-manager

Introduce storage manager concept

app/controllers/iqdb_queries_controller.rb

@@ -49,7 +49,7 @@ protected
 
   def create_by_post
     @post = Post.find(params[:post_id])
-    @download = Iqdb::Download.new(@post.complete_preview_file_url)
+    @download = Iqdb::Download.new(@post.preview_file_url)
     @download.find_similar
     @results = @download.matches
   end

app/controllers/uploads_controller.rb

@@ -6,7 +6,7 @@ class UploadsController < ApplicationController
     @upload = Upload.new
     @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
     if params[:url]
-      download = Downloads::File.new(params[:url], ".")
+      download = Downloads::File.new(params[:url])
       @normalized_url, _, _ = download.before_download(params[:url], {})
       @post = find_post_by_url(@normalized_url)
 

app/logical/apng_inspector.rb

@@ -86,7 +86,7 @@ class APNGInspector
     end
 
     @corrupted = !read_success || actl_corrupted
-    return !@corrupted
+    self
   end
 
   def corrupted?
@@ -109,4 +109,4 @@ class APNGInspector
       return framedata.unpack("N".freeze)[0]
     end
 
-end
+end

app/logical/backup_service.rb

@@ -1,9 +0,0 @@
-class BackupService
-  def backup(file_path, options = {})
-    raise NotImplementedError.new("#{self.class}.backup not implemented")
-  end
-
-  def delete(file_path, options = {})
-    raise NotImplementedError.new("#{self.class}.delete not implemented")
-  end
-end

app/logical/danbooru_image_resizer.rb

@@ -1,6 +1,6 @@
 module DanbooruImageResizer
-  def resize(read_path, write_path, width, height, resize_quality = 90)
+  def resize(file, width, height, resize_quality = 90)
-    image = Magick::Image.read(read_path).first
+    image = Magick::Image.read(file.path).first
     geometry = "#{width}x>"
 
     if width == Danbooru.config.small_image_width
@@ -17,14 +17,15 @@ module DanbooruImageResizer
     image = flatten(image, width, height)
     image.strip!
 
-    image.write(write_path) do
+    output_file = Tempfile.new(binmode: true)
+    image.write("jpeg:" + output_file.path) do
       self.quality = resize_quality
       # setting PlaneInterlace enables progressive encoding for JPEGs
       self.interlace = Magick::PlaneInterlace
     end
 
     image.destroy!
-    FileUtils.chmod(0664, write_path)
+    output_file
   end
 
   def flatten(image, width, height)

app/logical/downloads/file.rb

@@ -3,9 +3,9 @@ module Downloads
     class Error < Exception ; end
 
     attr_reader :data, :options
-    attr_accessor :source, :original_source, :downloaded_source, :file_path
+    attr_accessor :source, :original_source, :downloaded_source
 
-    def initialize(source, file_path, options = {})
+    def initialize(source, options = {})
       # source can potentially get rewritten in the course
       # of downloading a file, so check it again
       @source = source
@@ -14,9 +14,6 @@ module Downloads
       # the URL actually downloaded after rewriting the original source.
       @downloaded_source = nil
 
-      # where to save the download
-      @file_path = file_path
-
       # we sometimes need to capture data from the source page
       @data = {}
 
@@ -35,12 +32,13 @@ module Downloads
     def download!
       url, headers, @data = before_download(@source, @data)
 
-      ::File.open(@file_path, "wb") do |out|
+      output_file = Tempfile.new(binmode: true)
-        http_get_streaming(uncached_url(url, headers), out, headers)
+      http_get_streaming(uncached_url(url, headers), output_file, headers)
-      end
 
       @downloaded_source = url
       @source = after_download(url)
+
+      output_file
     end
 
     def before_download(url, datums)
@@ -91,7 +89,8 @@ module Downloads
           end
 
           if res.success?
-            return
+            file.rewind
+            return file
           else
             raise Error.new("HTTP error code: #{res.code} #{res.message}")
           end

app/logical/null_backup_service.rb

@@ -1,9 +0,0 @@
-class NullBackupService
-  def backup(file_path, options = {})
-    # do nothing
-  end
-
-  def delete(file_path, options = {})
-    # do nothing
-  end
-end

app/logical/pixiv_ugoira_converter.rb

@@ -1,13 +1,9 @@
 class PixivUgoiraConverter
-  def self.convert(source_path, output_path, preview_path, frame_data)
+  def self.generate_webm(ugoira_file, frame_data)
-    folder = Zip::File.new(source_path)
+    folder = Zip::File.new(ugoira_file.path)
-    write_webm(folder, output_path, frame_data)
+    output_file = Tempfile.new(binmode: true)
-    write_preview(folder, preview_path)
+    write_path = output_file.path
-    RemoteFileManager.new(output_path).distribute
-    RemoteFileManager.new(preview_path).distribute
-  end
 
-  def self.write_webm(folder, write_path, frame_data)
     Dir.mktmpdir do |tmpdir|
       FileUtils.mkdir_p("#{tmpdir}/images")
       folder.each_with_index do |file, i|
@@ -64,14 +60,17 @@ class PixivUgoiraConverter
         return
       end
     end
+
+    output_file
   end
 
-  def self.write_preview(folder, path)
+  def self.generate_preview(ugoira_file)
-    Dir.mktmpdir do |tmpdir|
+    file = Tempfile.new(binmode: true)
-      file = folder.first
+    zipfile = Zip::File.new(ugoira_file.path)
-      temp_path = File.join(tmpdir, file.name)
+    zipfile.entries.first.extract(file.path) { true } #  'true' means overwrite the existing tempfile.
-      file.extract(temp_path)
+
-      DanbooruImageResizer.resize(temp_path, path, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
+    DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
-    end
+  ensure
+    file.close!
   end
 end

app/logical/pixiv_ugoira_service.rb

@@ -1,32 +1,10 @@
 class PixivUgoiraService
   attr_reader :width, :height, :frame_data, :content_type
 
-  def self.regen(post)
-    service = new()
-    service.load(
-      :is_ugoira => true,
-      :ugoira_frame_data => post.pixiv_ugoira_frame_data.data
-    )
-    service.generate_resizes(post.file_path, post.large_file_path, post.preview_file_path, false)
-  end
-
   def save_frame_data(post)
     PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id)
   end
 
-  def generate_resizes(source_path, output_path, preview_path, delay = true)
-    # Run this a bit in the future to give the upload process time to move the file
-    if delay
-      PixivUgoiraConverter.delay(:queue => Socket.gethostname, :run_at => 10.seconds.from_now, :priority => -1).convert(source_path, output_path, preview_path, @frame_data)
-    else
-      PixivUgoiraConverter.convert(source_path, output_path, preview_path, @frame_data)
-    end
-
-    # since the resizes will be delayed, just touch the output file so the
-    # file distribution wont break
-    FileUtils.touch([output_path, preview_path])
-  end
-
   def calculate_dimensions(source_path)
     folder = Zip::File.new(source_path)
     tempfile = Tempfile.new("ugoira-dimensions")

app/logical/s3_backup_service.rb

@@ -1,52 +0,0 @@
-class S3BackupService < BackupService
-  attr_reader :client, :bucket
-
-  def initialize(client: nil, bucket: Danbooru.config.aws_s3_bucket_name)
-    @credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
-    @client = client || Aws::S3::Client.new(credentials: @credentials, region: "us-east-1", logger: Logger.new(STDOUT))
-    @bucket = bucket
-  end
-
-  def backup(file_path, type: nil, **options)
-    keys = s3_keys(file_path, type)
-    keys.each do |key|
-      upload_to_s3(key, file_path)
-    end
-  end
-
-  def delete(file_path, type: nil)
-    keys = s3_keys(file_path, type)
-    keys.each do |key|
-      delete_from_s3(key)
-    end
-  end
-
-protected
-  def s3_keys(file_path, type)
-    name = File.basename(file_path)
-
-    case type
-    when :original
-      [name]
-    when :preview
-      ["preview/#{name}"]
-    when :large
-      ["sample/#{name}"]
-    else
-      raise ArgumentError.new("Unknown type: #{type}")
-    end
-  end
-
-  def delete_from_s3(key)
-    client.delete_object(bucket: bucket, key: key)
-  rescue Aws::S3::Errors::NoSuchKey
-    # ignore
-  end
-
-  def upload_to_s3(key, file_path)
-    File.open(file_path, "rb") do |body|
-      base64_md5 = Digest::MD5.base64digest(File.read(file_path))
-      client.put_object(acl: "public-read", bucket: bucket, key: key, body: body, content_md5: base64_md5)
-    end
-  end
-end

app/logical/storage_manager.rb

@@ -0,0 +1,106 @@
+class StorageManager
+  class Error < StandardError; end
+
+  DEFAULT_BASE_URL = Rails.application.routes.url_helpers.root_url + "data"
+  DEFAULT_BASE_DIR = "#{Rails.root}/public/data"
+
+  attr_reader :base_url, :base_dir, :hierarchical, :tagged_filenames, :large_image_prefix
+
+  def initialize(base_url: DEFAULT_BASE_URL, base_dir: DEFAULT_BASE_DIR, hierarchical: false, tagged_filenames: Danbooru.config.enable_seo_post_urls, large_image_prefix: Danbooru.config.large_image_prefix)
+    @base_url = base_url.chomp("/")
+    @base_dir = base_dir
+    @hierarchical = hierarchical
+    @tagged_filenames = tagged_filenames
+    @large_image_prefix = large_image_prefix
+  end
+
+  # Store the given file at the given path. If a file already exists at that
+  # location it should be overwritten atomically. Either the file is fully
+  # written, or an error is raised and the original file is left unchanged. The
+  # file should never be in a partially written state.
+  def store(io, path)
+    raise NotImplementedError, "store not implemented"
+  end
+
+  # Delete the file at the given path. If the file doesn't exist, no error
+  # should be raised.
+  def delete(path)
+    raise NotImplementedError, "delete not implemented"
+  end
+
+  # Return a readonly copy of the file located at the given path.
+  def open(path)
+    raise NotImplementedError, "open not implemented"
+  end
+
+  def store_file(io, post, type)
+    store(io, file_path(post.md5, post.file_ext, type))
+  end
+
+  def delete_file(post_id, md5, file_ext, type)
+    delete(file_path(md5, file_ext, type))
+  end
+
+  def open_file(post, type)
+    open(file_path(post.md5, post.file_ext, type))
+  end
+
+  def file_url(post, type)
+    subdir = subdir_for(post.md5)
+    file = file_name(post.md5, post.file_ext, type)
+
+    if type == :preview && !post.has_preview?
+      "#{base_url}/images/download-preview.png"
+    elsif type == :preview
+      "#{base_url}/preview/#{subdir}#{file}"
+    elsif type == :large && post.has_large?
+      "#{base_url}/sample/#{subdir}#{seo_tags(post)}#{file}"
+    else
+      "#{base_url}/#{subdir}#{seo_tags(post)}#{file}"
+    end
+  end
+
+  protected
+
+  def file_path(md5, file_ext, type)
+    subdir = subdir_for(md5)
+    file = file_name(md5, file_ext, type)
+
+    case type
+    when :preview
+      "#{base_dir}/preview/#{subdir}#{file}"
+    when :large
+      "#{base_dir}/sample/#{subdir}#{file}"
+    when :original
+      "#{base_dir}/#{subdir}#{file}"
+    end
+  end
+
+  def file_name(md5, file_ext, type)
+    large_file_ext = (file_ext == "zip") ? "webm" : "jpg"
+
+    case type
+    when :preview
+      "#{md5}.jpg"
+    when :large
+      "#{large_image_prefix}#{md5}.#{large_file_ext}"
+    when :original
+      "#{md5}.#{file_ext}"
+    end
+  end
+
+  def subdir_for(md5)
+    if hierarchical
+      "#{md5[0..1]}/#{md5[2..3]}/"
+    else
+      ""
+    end
+  end
+
+  def seo_tags(post, user = CurrentUser.user)
+    return "" if !tagged_filenames || user.disable_tagged_filenames?
+
+    tags = post.humanized_essential_tag_string.gsub(/[^a-z0-9]+/, "_").gsub(/(?:^_+)|(?:_+$)/, "").gsub(/_{2,}/, "_")
+    "__#{tags}__"
+  end
+end

app/logical/storage_manager/hybrid.rb

@@ -0,0 +1,23 @@
+class StorageManager::Hybrid < StorageManager
+  attr_reader :submanager
+
+  def initialize(&block)
+    @submanager = block
+  end
+
+  def store_file(io, post, type)
+    submanager[post.id, post.md5, post.file_ext, type].store_file(io, post, type)
+  end
+
+  def delete_file(post_id, md5, file_ext, type)
+    submanager[post_id, md5, file_ext, type].delete_file(post_id, md5, file_ext, type)
+  end
+
+  def open_file(io, post, type)
+    submanager[post.id, post.md5, post.file_ext, type].open_file(post, type)
+  end
+
+  def file_url(post, type)
+    submanager[post.id, post.md5, post.file_ext, type].file_url(post, type)
+  end
+end

app/logical/storage_manager/local.rb

@@ -0,0 +1,25 @@
+class StorageManager::Local < StorageManager
+  DEFAULT_PERMISSIONS = 0644
+
+  def store(io, dest_path)
+    temp_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
+
+    FileUtils.mkdir_p(File.dirname(temp_path))
+    bytes_copied = IO.copy_stream(io, temp_path)
+    raise Error, "store failed: #{bytes_copied}/#{io.size} bytes copied" if bytes_copied != io.size
+
+    FileUtils.chmod(DEFAULT_PERMISSIONS, temp_path)
+    File.rename(temp_path, dest_path)
+  rescue StandardError => e
+    FileUtils.rm_f(temp_path)
+    raise Error, e
+  end
+
+  def delete(path)
+    FileUtils.rm_f(path)
+  end
+
+  def open(path)
+    File.open(path, "r", binmode: true)
+  end
+end

app/logical/storage_manager/null.rb

@@ -0,0 +1,13 @@
+class StorageManager::Null < StorageManager
+  def store(io, path)
+    # no-op
+  end
+
+  def delete(path)
+    # no-op
+  end
+
+  def open(path)
+    # no-op
+  end
+end

app/logical/storage_manager/s3.rb

@@ -0,0 +1,43 @@
+class StorageManager::S3 < StorageManager
+  # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method
+  DEFAULT_S3_OPTIONS = {
+    region: Danbooru.config.aws_region,
+    credentials: Danbooru.config.aws_credentials,
+    logger: Rails.logger,
+  }
+
+  # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#put_object-instance_method
+  DEFAULT_PUT_OPTIONS = {
+    acl: "public-read",
+    storage_class: "STANDARD", # STANDARD, STANDARD_IA, REDUCED_REDUNDANCY
+    cache_control: "public, max-age=#{1.year.to_i}",
+    #content_type: "image/jpeg" # XXX should set content type
+  }
+
+  attr_reader :bucket, :client, :s3_options
+
+  def initialize(bucket, client: nil, s3_options: {}, **options)
+    @bucket = bucket
+    @s3_options = DEFAULT_S3_OPTIONS.merge(s3_options)
+    @client = client || Aws::S3::Client.new(**@s3_options)
+    super(**options)
+  end
+
+  def store(io, path)
+    data = io.read
+    base64_md5 = Digest::MD5.base64digest(data)
+    client.put_object(bucket: bucket, key: path, body: data, content_md5: base64_md5, **DEFAULT_PUT_OPTIONS)
+  end
+
+  def delete(path)
+    client.delete_object(bucket: bucket, key: path)
+  rescue Aws::S3::Errors::NoSuchKey
+    # ignore
+  end
+
+  def open(path)
+    file = Tempfile.new(binmode: true)
+    client.get_object(bucket: bucket: key: path, response_target: file)
+    file
+  end
+end

app/logical/storage_manager/sftp.rb

@@ -0,0 +1,76 @@
+class StorageManager::SFTP < StorageManager
+  DEFAULT_PERMISSIONS = 0644
+
+  # http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start
+  DEFAULT_SSH_OPTIONS = {
+    timeout: 10,
+    logger: Rails.logger,
+    verbose: :fatal,
+    non_interactive: true,
+  }
+
+  attr_reader :hosts, :ssh_options
+
+  def initialize(*hosts, ssh_options: {}, **options)
+    @hosts = hosts
+    @ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options)
+    super(**options)
+  end
+
+  def store(file, dest_path)
+    temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
+    dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak"
+
+    each_host do |host, sftp|
+      begin
+        sftp.upload!(file.path, temp_upload_path)
+        sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS)
+
+        # `rename!` can't overwrite existing files, so if a file already exists
+        # at dest_path we move it out of the way first.
+        force { sftp.rename!(dest_path, dest_backup_path) }
+        force { sftp.rename!(temp_upload_path, dest_path) }
+      rescue StandardError => e
+        # if anything fails, try to move the original file back in place (if it was moved).
+        force { sftp.rename!(dest_backup_path, dest_path) }
+        raise Error, e
+      ensure
+        force { sftp.remove!(temp_upload_path) }
+        force { sftp.remove!(dest_backup_path) }
+      end
+    end
+  end
+
+  def delete(dest_path)
+    each_host do |host, sftp|
+      force { sftp.remove!(dest_path) }
+    end
+  end
+
+  def open(dest_path)
+    file = Tempfile.new(binmode: true)
+
+    Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp|
+      sftp.download!(dest_path, file.path)
+    end
+
+    file
+  end
+
+  protected
+
+  # Ignore "no such file" exceptions for the given operation.
+  def force
+    yield
+  rescue Net::SFTP::StatusException => e
+    raise Error, e unless e.description == "no such file"
+  end
+
+  def each_host
+    hosts.each do |host|
+      Net::SFTP.start(host, nil, ssh_options) do |sftp|
+        yield host, sftp
+      end
+    end
+  end
+end

app/models/post.rb

@@ -29,7 +29,6 @@ class Post < ApplicationRecord
   before_save :set_tag_counts
   before_save :set_pool_category_pseudo_tags
   before_create :autoban
-  after_save :queue_backup, if: :md5_changed?
   after_save :create_version
   after_save :update_parent_on_save
   after_save :apply_post_metatags
@@ -94,144 +93,76 @@ class Post < ApplicationRecord
     extend ActiveSupport::Concern
 
     module ClassMethods
-      def delete_files(post_id, file_path, large_file_path, preview_file_path, force: false)
+      def delete_files(post_id, md5, file_ext, force: false)
-        unless force
+        if Post.where(md5: md5).exists? && !force
-          # XXX should pass in the md5 instead of parsing it.
+          raise DeletionError.new("Files still in use; skipping deletion.")
-          preview_file_path =~ %r!/data/preview/(?:test\.)?([a-z0-9]{32})\.jpg\z!
-          md5 = $1
-
-          if Post.where(md5: md5).exists?
-            raise DeletionError.new("Files still in use; skipping deletion.")
-          end
         end
 
-        backup_service = Danbooru.config.backup_service
+        Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :original)
-        backup_service.delete(file_path, type: :original)
+        Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :large)
-        backup_service.delete(large_file_path, type: :large)
+        Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :preview)
-        backup_service.delete(preview_file_path, type: :preview)
 
-        # the large file and the preview don't necessarily exist. if so errors will be ignored.
+        Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :original)
-        FileUtils.rm_f(file_path)
+        Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :large)
-        FileUtils.rm_f(large_file_path)
+        Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :preview)
-        FileUtils.rm_f(preview_file_path)
-
-        RemoteFileManager.new(file_path).delete
-        RemoteFileManager.new(large_file_path).delete
-        RemoteFileManager.new(preview_file_path).delete
 
         if Danbooru.config.cloudflare_key
-          md5, ext = File.basename(file_path).split(".")
+          CloudflareService.new.delete(md5, file_ext)
-          CloudflareService.new.delete(md5, ext)
         end
       end
     end
 
+    def queue_delete_files(grace_period)
+      Post.delay(queue: "default", run_at: Time.now + grace_period).delete_files(id, md5, file_ext)
+    end
+
     def delete_files
-      Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true)
+      Post.delete_files(id, md5, file_ext, force: true)
     end
 
-    def distribute_files
+    def distribute_files(file, sample_file, preview_file)
-      if Danbooru.config.build_file_url(self) =~ /^http/
+      storage_manager.store_file(file, self, :original)
-        # this post is archived
+      storage_manager.store_file(sample_file, self, :large) if sample_file.present?
-        RemoteFileManager.new(file_path).distribute_to_archive(Danbooru.config.build_file_url(self))
+      storage_manager.store_file(preview_file, self, :preview) if preview_file.present?
-        RemoteFileManager.new(preview_file_path).distribute if has_preview?
+
-        RemoteFileManager.new(large_file_path).distribute_to_archive(Danbooru.config.build_large_file_url(self)) if has_large?
+      backup_storage_manager.store_file(file, self, :original)
-      else
+      backup_storage_manager.store_file(sample_file, self, :large) if sample_file.present?
-        RemoteFileManager.new(file_path).distribute
+      backup_storage_manager.store_file(preview_file, self, :preview) if preview_file.present?
-        RemoteFileManager.new(preview_file_path).distribute if has_preview?
-        RemoteFileManager.new(large_file_path).distribute if has_large?
-      end
     end
 
-    def file_path_prefix
+    def backup_storage_manager
-      Rails.env == "test" ? "test." : ""
+      Danbooru.config.backup_storage_manager
     end
 
-    def file_path
+    def storage_manager
-      "#{Rails.root}/public/data/#{file_path_prefix}#{md5}.#{file_ext}"
+      Danbooru.config.storage_manager
     end
 
-    def large_file_path
+    def file(type = :original)
-      if has_large?
+      storage_manager.open_file(self, type)
-        "#{Rails.root}/public/data/sample/#{file_path_prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}"
-      else
-        file_path
-      end
-    end
-
-    def large_file_ext
-      if is_ugoira?
-        "webm"
-      else
-        "jpg"
-      end
-    end
-
-    def preview_file_path
-      "#{Rails.root}/public/data/preview/#{file_path_prefix}#{md5}.jpg"
-    end
-
-    def file_name
-      "#{file_path_prefix}#{md5}.#{file_ext}"
     end
 
     def file_url
-      Danbooru.config.build_file_url(self)
+      storage_manager.file_url(self, :original)
-    end
-
-    # this is for the 640x320 version
-    def cropped_file_url
     end
 
     def large_file_url
-      if has_large?
+      storage_manager.file_url(self, :large)
-        Danbooru.config.build_large_file_url(self)
-      else
-        file_url
-      end
-    end
-
-    def seo_tag_string
-      if Danbooru.config.enable_seo_post_urls && !CurrentUser.user.disable_tagged_filenames?
-        "__#{seo_tags}__"
-      else
-        nil
-      end
-    end
-
-    def seo_tags
-      @seo_tags ||= humanized_essential_tag_string.gsub(/[^a-z0-9]+/, "_").gsub(/(?:^_+)|(?:_+$)/, "").gsub(/_{2,}/, "_")
     end
 
     def preview_file_url
-      if !has_preview?
+      storage_manager.file_url(self, :preview)
-        return "/images/download-preview.png"
-      end
-
-      "/data/preview/#{file_path_prefix}#{md5}.jpg"
-    end
-
-    def complete_preview_file_url
-      "http://#{Danbooru.config.hostname}#{preview_file_url}"
     end
 
     def open_graph_image_url
       if is_image?
         if has_large?
-          if Danbooru.config.build_large_file_url(self) =~ /http/
+          large_file_url
-            large_file_url
-          else
-            "http://#{Danbooru.config.hostname}#{large_file_url}"
-          end
         else
-          if Danbooru.config.build_file_url(self) =~ /http/
+          file_url
-            file_url
-          else
-            "http://#{Danbooru.config.hostname}#{file_url}"
-          end
         end
       else
-        complete_preview_file_url
+        preview_file_url
       end
     end
 
@@ -243,36 +174,10 @@ class Post < ApplicationRecord
       end
     end
 
-    def file_path_for(user)
-      if user.default_image_size == "large" && image_width > Danbooru.config.large_image_width
-        large_file_path
-      else
-        file_path
-      end
-    end
-
     def is_image?
       file_ext =~ /jpg|jpeg|gif|png/i
     end
 
-    def is_animated_gif?
-      if file_ext =~ /gif/i && File.exists?(file_path)
-        return Magick::Image.ping(file_path).length > 1
-      else
-        return false
-      end
-    end
-    
-    def is_animated_png?
-      if file_ext =~ /png/i && File.exists?(file_path)
-        apng = APNGInspector.new(file_path)
-        apng.inspect!
-        return apng.animated?
-      else
-        return false
-      end
-    end
-
     def is_flash?
       file_ext =~ /swf/i
     end
@@ -294,9 +199,7 @@ class Post < ApplicationRecord
     end
 
     def has_preview?
-      # for video/ugoira we don't want to try and render a preview that
+      is_image? || is_video? || is_ugoira?
-      # might doesn't exist yet
-      is_image? || ((is_video? || is_ugoira?) && File.exists?(preview_file_path))
     end
 
     def has_dimensions?
@@ -304,24 +207,7 @@ class Post < ApplicationRecord
     end
 
     def has_ugoira_webm?
-      created_at < 1.minute.ago || (File.exists?(preview_file_path) && File.size(preview_file_path) > 0)
+      true
-    end
-  end
-
-  module BackupMethods
-    extend ActiveSupport::Concern
-
-    def queue_backup
-      Post.delay(queue: "default", priority: -1).backup_file(file_path, id: id, type: :original)
-      Post.delay(queue: "default", priority: -1).backup_file(large_file_path, id: id, type: :large) if has_large?
-      Post.delay(queue: "default", priority: -1).backup_file(preview_file_path, id: id, type: :preview) if has_preview?
-    end
-
-    module ClassMethods
-      def backup_file(file_path, options = {})
-        backup_service = Danbooru.config.backup_service
-        backup_service.backup(file_path, options)
-      end
     end
   end
 
@@ -765,7 +651,6 @@ class Post < ApplicationRecord
       return tags if !Danbooru.config.enable_dimension_autotagging
 
       tags -= %w(incredibly_absurdres absurdres highres lowres huge_filesize flash webm mp4)
-      tags -= %w(animated_gif animated_png) if new_record?
 
       if has_dimensions?
         if image_width >= 10_000 || image_height >= 10_000
@@ -794,14 +679,6 @@ class Post < ApplicationRecord
         tags << "huge_filesize"
       end
 
-      if is_animated_gif?
-        tags << "animated_gif"
-      end
-      
-      if is_animated_png?
-        tags << "animated_png"
-      end
-
       if is_flash?
         tags << "flash"
       end
@@ -1747,8 +1624,8 @@ class Post < ApplicationRecord
     end
 
     def update_iqdb_async
-      if File.exists?(preview_file_path) && Post.iqdb_enabled?
+      if Post.iqdb_enabled?
-        Post.iqdb_sqs_service.send_message("update\n#{id}\n#{complete_preview_file_url}")
+        Post.iqdb_sqs_service.send_message("update\n#{id}\n#{preview_file_url}")
       end
     end
 
@@ -1854,7 +1731,6 @@ class Post < ApplicationRecord
   end
   
   include FileMethods
-  include BackupMethods
   include ImageMethods
   include ApprovalMethods
   include PresenterMethods

app/models/post_replacement.rb

@@ -24,6 +24,8 @@ class PostReplacement < ApplicationRecord
   end
 
   def process!
+    upload = nil
+
     transaction do
       upload = Upload.create!(
         file: replacement_file,
@@ -47,7 +49,7 @@ class PostReplacement < ApplicationRecord
       # md5/file_ext to delete the old files. if saving the post fails,
       # this is rolled back so the job won't run.
       if md5_changed
-        Post.delay(queue: "default", run_at: Time.now + DELETION_GRACE_PERIOD).delete_files(post.id, post.file_path, post.large_file_path, post.preview_file_path)
+        post.queue_delete_files(DELETION_GRACE_PERIOD)
       end
 
       self.file_ext = upload.file_ext
@@ -69,8 +71,6 @@ class PostReplacement < ApplicationRecord
 
       if md5_changed
         post.comments.create!({creator: User.system, body: comment_replacement_message, do_not_bump_post: true}, without_protection: true)
-      else
-        post.queue_backup
       end
 
       save!
@@ -79,7 +79,7 @@ class PostReplacement < ApplicationRecord
 
     # point of no return: these things can't be rolled back, so we do them
     # only after the transaction successfully commits.
-    post.distribute_files
+    upload.distribute_files(post)
     post.update_iqdb_async
   end
 

app/models/upload.rb

@@ -10,14 +10,11 @@ class Upload < ApplicationRecord
   belongs_to :uploader, :class_name => "User"
   belongs_to :post
   before_validation :initialize_uploader, :on => :create
-  before_validation :initialize_status, :on => :create
-  before_create :convert_cgi_file
-  after_destroy :delete_temp_file
   validate :uploader_is_not_limited, :on => :create
   validate :file_or_source_is_present, :on => :create
   validate :rating_given
   attr_accessible :file, :image_width, :image_height, :file_ext, :md5, 
-    :file_size, :as_pending, :source, :file_path, :content_type, :rating, 
+    :file_size, :as_pending, :source, :rating,
     :tag_string, :status, :backtrace, :post_id, :md5_confirmation, 
     :parent_id, :server, :artist_commentary_title,
     :artist_commentary_desc, :include_artist_commentary,
@@ -53,12 +50,6 @@ class Upload < ApplicationRecord
       end
     end
 
-    def validate_file_exists
-      unless file_path && File.exists?(file_path)
-        raise "file does not exist"
-      end
-    end
-
     def validate_file_content_type
       unless is_valid_content_type?
         raise "invalid content type (only JPEG, PNG, GIF, SWF, MP4, and WebM files are allowed)"
@@ -75,12 +66,6 @@ class Upload < ApplicationRecord
       end
     end
 
-    def validate_md5_confirmation_after_move
-      if !md5_confirmation.blank? && md5_confirmation != Digest::MD5.file(md5_file_path).hexdigest
-        raise "md5 mismatch"
-      end
-    end
-
     def rating_given
       if rating.present?
         return true
@@ -93,10 +78,14 @@ class Upload < ApplicationRecord
       end
     end
 
-    def tag_audio
+    def automatic_tags
-      if is_video? && video.audio_channels.present?
+      return "" unless Danbooru.config.enable_dimension_autotagging
-        self.tag_string = "#{tag_string} video_with_sound"
+
-      end
+      tags = []
+      tags << "video_with_sound" if is_video_with_audio?
+      tags << "animated_gif" if is_animated_gif?
+      tags << "animated_png" if is_animated_png?
+      tags.join(" ")
     end
 
     def validate_video_duration
@@ -110,35 +99,36 @@ class Upload < ApplicationRecord
 
   module ConversionMethods
     def process_upload
-      CurrentUser.scoped(uploader, uploader_ip_addr) do
+      begin
         update_attribute(:status, "processing")
-        self.source = strip_source
+
+        self.source = source.to_s.strip
         if is_downloadable?
-          self.downloaded_source, self.source = download_from_source(temp_file_path)
+          self.downloaded_source, self.source, self.file = download_from_source(source, referer_url)
+        else
+          self.file = self.file.tempfile
         end
-        validate_file_exists
+
-        self.content_type = file_header_to_content_type(file_path)
+        self.file_ext = file_header_to_file_ext(file)
-        self.file_ext = content_type_to_file_ext(content_type)
+        self.file_size = file.size
+        self.md5 = Digest::MD5.file(file.path).hexdigest
+
         validate_file_content_type
-        calculate_hash(file_path)
         validate_md5_uniqueness
         validate_md5_confirmation
-        tag_audio
         validate_video_duration
-        calculate_file_size(file_path)
+
-        if has_dimensions?
+        self.tag_string = "#{tag_string} #{automatic_tags}"
-          calculate_dimensions(file_path)
+        self.image_width, self.image_height = calculate_dimensions
-        end
+
-        generate_resizes(file_path)
-        move_file
-        validate_md5_confirmation_after_move
         save
       end
     end
 
     def create_post_from_upload
       post = convert_to_post
-      post.distribute_files
+      distribute_files(post)
+
       if post.save
         create_artist_commentary(post) if include_artist_commentary?
         ugoira_service.save_frame_data(post) if is_ugoira?
@@ -151,6 +141,14 @@ class Upload < ApplicationRecord
       post
     end
 
+    def distribute_files(post)
+      preview_file, sample_file = generate_resizes
+      post.distribute_files(file, sample_file, preview_file)
+    ensure
+      preview_file.try(:close!)
+      sample_file.try(:close!)
+    end
+
     def process!(force = false)
       @tries ||= 0
       return if !force && status =~ /processing|completed|error/
@@ -170,13 +168,9 @@ class Upload < ApplicationRecord
     rescue Exception => x
       update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
       nil
-      
-    ensure
-      delete_temp_file
-    end
 
-    def async_conversion?
+    ensure
-      is_ugoira?
+      file.try(:close!)
     end
 
     def ugoira_service
@@ -211,23 +205,6 @@ class Upload < ApplicationRecord
   end
 
   module FileMethods
-    def delete_temp_file(path = nil)
-      FileUtils.rm_f(path || temp_file_path)
-    end
-
-    def move_file
-      FileUtils.mv(file_path, md5_file_path)
-    end
-
-    def calculate_file_size(source_path)
-      self.file_size = File.size(source_path)
-    end
-
-    # Calculates the MD5 based on whatever is in temp_file_path
-    def calculate_hash(source_path)
-      self.md5 = Digest::MD5.file(source_path).hexdigest
-    end
-
     def is_image?
       %w(jpg gif png).include?(file_ext)
     end
@@ -240,75 +217,68 @@ class Upload < ApplicationRecord
       %w(webm mp4).include?(file_ext)
     end
 
+    def is_video_with_audio?
+      is_video? && video.audio_channels.present?
+    end
+
     def is_ugoira?
       %w(zip).include?(file_ext)
     end
+
+    def is_animated_gif?
+      file_ext == "gif" && Magick::Image.ping(file.path).length > 1
+    end
+
+    def is_animated_png?
+      file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
+    end
   end
 
   module ResizerMethods
-    def generate_resizes(source_path)
+    def generate_resizes
-      generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85)
+      if is_video?
+        preview_file = generate_video_preview_for(video, width, height)
+      elsif is_ugoira?
+        preview_file = PixivUgoiraConverter.generate_preview(file)
+        sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data)
+      elsif is_image?
+        preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
 
-      if is_image? && image_width > Danbooru.config.large_image_width
+        if image_width > Danbooru.config.large_image_width
-        generate_resize_for(Danbooru.config.large_image_width, nil, source_path)
+          sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, nil, 90)
+        end
       end
+
+      [preview_file, sample_file]
     end
 
-    def generate_video_preview_for(width, height, output_path)
+    def generate_video_preview_for(video, width, height)
-      dimension_ratio = image_width.to_f / image_height
+      dimension_ratio = video.width.to_f / video.height
       if dimension_ratio > 1
         height = (width / dimension_ratio).to_i
       else
         width = (height * dimension_ratio).to_i
       end
-      video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
-      FileUtils.chmod(0664, output_path)
-    end
 
-    def generate_resize_for(width, height, source_path, quality = 90)
+      output_file = Tempfile.new(binmode: true)
-      unless File.exists?(source_path)
+      video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
-        raise Error.new("file not found")
+      output_file
-      end
-
-      output_path = resized_file_path_for(width)
-      if is_image?
-        DanbooruImageResizer.resize(source_path, output_path, width, height, quality)
-      elsif is_ugoira?
-        if Delayed::Worker.delay_jobs
-          # by the time this runs we'll have moved source_path to md5_file_path
-          ugoira_service.generate_resizes(md5_file_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width))
-        else
-          ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width), false)
-        end
-      elsif is_video?
-        generate_video_preview_for(width, height, output_path)
-      end
     end
   end
 
   module DimensionMethods
     # Figures out the dimensions of the image.
-    def calculate_dimensions(file_path)
+    def calculate_dimensions
       if is_video?
-        self.image_width = video.width
+        [video.width, video.height]
-        self.image_height = video.height
       elsif is_ugoira?
-        ugoira_service.calculate_dimensions(file_path)
+        ugoira_service.calculate_dimensions(file.path)
-        self.image_width = ugoira_service.width
+        [ugoira_service.width, ugoira_service.height]
-        self.image_height = ugoira_service.height
       else
-        File.open(file_path, "rb") do |file|
+        image_size = ImageSpec.new(file.path)
-          image_size = ImageSpec.new(file)
+        [image_size.width, image_size.height]
-          self.image_width = image_size.width
-          self.image_height = image_size.height
-        end
       end
     end
-
-    # Does this file have image dimensions?
-    def has_dimensions?
-      %w(jpg gif png swf webm mp4 zip).include?(file_ext)
-    end
   end
 
   module ContentTypeMethods
@@ -316,136 +286,44 @@ class Upload < ApplicationRecord
       file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/
     end
 
-    def content_type_to_file_ext(content_type)
+    def file_header_to_file_ext(file)
-      case content_type
+      case File.read(file.path, 16)
-      when "image/jpeg"
+      when /^\xff\xd8/n
         "jpg"
-
+      when /^GIF87a/, /^GIF89a/
-      when "image/gif"
         "gif"
-
+      when /^\x89PNG\r\n\x1a\n/n
-      when "image/png"
         "png"
-
+      when /^CWS/, /^FWS/, /^ZWS/
-      when "application/x-shockwave-flash"
         "swf"
-
+      when /^\x1a\x45\xdf\xa3/n
-      when "video/webm"
         "webm"
-
+      when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
-      when "video/mp4"
         "mp4"
-
+      when /^PK\x03\x04/
-      when "application/zip"
         "zip"
-
       else
         "bin"
       end
     end
-
-    def file_header_to_content_type(source_path)
-      case File.read(source_path, 16)
-      when /^\xff\xd8/n
-        "image/jpeg"
-
-      when /^GIF87a/, /^GIF89a/
-        "image/gif"
-
-      when /^\x89PNG\r\n\x1a\n/n
-        "image/png"
-
-      when /^CWS/, /^FWS/, /^ZWS/
-        "application/x-shockwave-flash"
-
-      when /^\x1a\x45\xdf\xa3/n
-        "video/webm"
-
-      when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
-        "video/mp4"
-
-      when /^PK\x03\x04/
-        "application/zip"
-
-      else
-        "application/octet-stream"
-      end
-    end
-  end
-
-  module FilePathMethods
-    def md5_file_path
-      prefix = Rails.env == "test" ? "test." : ""
-      "#{Rails.root}/public/data/#{prefix}#{md5}.#{file_ext}"
-    end
-
-    def resized_file_path_for(width)
-      prefix = Rails.env == "test" ? "test." : ""
-
-      case width
-      when Danbooru.config.small_image_width
-        "#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg"
-
-      when Danbooru.config.large_image_width
-        "#{Rails.root}/public/data/sample/#{prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}"
-      end
-    end
-
-    def large_file_ext
-      if is_ugoira?
-        "webm"
-      else
-        "jpg"
-      end
-    end
-
-    def temp_file_path
-      @temp_file_path ||= File.join(Rails.root, "tmp", "upload_#{Time.now.to_f}.#{Process.pid}")
-    end
   end
 
   module DownloaderMethods
-    def strip_source
-      source.to_s.strip
-    end
-
     # Determines whether the source is downloadable
     def is_downloadable?
-      source =~ /^https?:\/\// && file_path.blank?
+      source =~ /^https?:\/\// && file.blank?
     end
 
-    # Downloads the file to destination_path
+    def download_from_source(source, referer_url)
-    def download_from_source(destination_path)
+      download = Downloads::File.new(source, referer_url: referer_url)
-      self.file_path = destination_path
+      file = download.download!
-      download = Downloads::File.new(source, destination_path, :referer_url => referer_url)
-      download.download!
       ugoira_service.load(download.data)
-      [download.downloaded_source, download.source]
-    end
-  end
 
-  module CgiFileMethods
+      [download.downloaded_source, download.source, file]
-    def convert_cgi_file
-      return if file.blank? || file.size == 0
-
-      self.file_path = temp_file_path
-
-      if file.respond_to?(:tempfile) && file.tempfile
-        FileUtils.cp(file.tempfile.path, file_path)
-      else
-        File.open(file_path, 'wb') do |out|
-          out.write(file.read)
-        end
-      end
-      FileUtils.chmod(0664, file_path)
     end
   end
 
   module StatusMethods
-    def initialize_status
-      self.status = "pending"
-    end
-
     def is_pending?
       status == "pending"
     end
@@ -480,7 +358,7 @@ class Upload < ApplicationRecord
 
   module VideoMethods
     def video
-      @video ||= FFMPEG::Movie.new(file_path)
+      @video ||= FFMPEG::Movie.new(file.path)
     end
   end
 
@@ -534,8 +412,6 @@ class Upload < ApplicationRecord
   include DimensionMethods
   include ContentTypeMethods
   include DownloaderMethods
-  include FilePathMethods
-  include CgiFileMethods
   include StatusMethods
   include UploaderMethods
   include VideoMethods

app/presenters/post_presenter.rb

@@ -135,6 +135,10 @@ class PostPresenter < Presenter
     @post.humanized_essential_tag_string
   end
 
+  def filename_for_download
+    "#{humanized_essential_tag_string} - #{@post.md5}.#{@post.file_ext}"
+  end
+
   def categorized_tag_groups
     string = []
 

app/views/comments/index.atom.builder

@@ -11,7 +11,7 @@ atom_feed(root_url: comments_url(host: Danbooru.config.hostname)) do |feed|
     feed.entry(comment, published: comment.created_at, updated: comment.updated_at) do |entry|
       entry.title("@#{comment.creator_name} on post ##{comment.post_id} (#{comment.post.humanized_essential_tag_string})")
       entry.content(<<-EOS.strip_heredoc, type: "html")
-        <img src="#{comment.post.complete_preview_file_url}"/>
+        <img src="#{comment.post.preview_file_url}"/>
 
         #{format_text(comment.body)}
       EOS

app/views/posts/partials/show/_options.html.erb

@@ -4,7 +4,7 @@
   <% if CurrentUser.is_member? %>
     <li><%= link_to "Favorite", favorites_path(:post_id => post.id), :remote => true, :method => :post, :id => "add-to-favorites", :title => "Shortcut is F" %></li>
     <li><%= link_to "Unfavorite", favorite_path(post), :remote => true, :method => :delete, :id => "remove-from-favorites" %></li>
-    <li><%= link_to_if post.visible?, "Download", post.file_url, :download => post.presenter.humanized_essential_tag_string + " - " + post.file_name %></li>
+    <li><%= link_to_if post.visible?, "Download", post.file_url, download: post.presenter.filename_for_download %></li>
     <li id="add-to-pool-list"><%= link_to "Add to pool", "#", :id => "pool" %></li>
     <% if post.is_note_locked? %>
       <li id="add-notes-list"><span id="note-locked-notice">Note locked</span></li>

config/danbooru_default_config.rb

@@ -99,20 +99,6 @@ module Danbooru
       true
     end
 
-    # What method to use to backup images.
-    #
-    # NullBackupService: Don't backup images at all.
-    #
-    # S3BackupService: Backup to Amazon S3. Must configure aws_access_key_id,
-    # aws_secret_access_key, and aws_s3_bucket_name. Bucket must exist and be writable.
-    def backup_service
-      if Rails.env.production?
-        S3BackupService.new
-      else
-        NullBackupService.new
-      end
-    end
-
     # What method to use to store images.
     # local_flat: Store every image in one directory.
     # local_hierarchy: Store every image in a hierarchical directory, based on the post's MD5 hash. On some file systems this may be faster.
@@ -222,12 +208,58 @@ module Danbooru
       "danbooru"
     end
 
-    def build_file_url(post)
+    # The method to use for storing image files.
-      "/data/#{post.file_path_prefix}/#{post.md5}.#{post.file_ext}"
+    def storage_manager
+      # Store files on the local filesystem.
+      # base_dir - where to store files (default: under public/data)
+      # base_url - where to serve files from (default: http://#{hostname}/data)
+      # hierarchical: false - store files in a single directory
+      # hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash
+      StorageManager::Local.new(base_dir: "#{Rails.root}/public/data", hierarchical: false)
+
+      # Store files on one or more remote host(s). Configure SSH settings in
+      # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
+      # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {})
+
+      # Store files in an S3 bucket. The bucket must already exist and be
+      # writable by you. Configure your S3 settings in aws_region and
+      # aws_credentials below, or in the s3_options param (ref:
+      # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method)
+      # StorageManager::S3.new("my_s3_bucket", base_url: "https://my_s3_bucket.s3.amazonaws.com/", s3_options: {})
+
+      # Select the storage method based on the post's id and type (preview, large, or original).
+      # StorageManager::Hybrid.new do |id, md5, file_ext, type|
+      #   ssh_options = { user: "danbooru" }
+      #
+      #   if type.in?([:large, :original]) && id.in?(0..850_000)
+      #     StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
+      #   elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000)
+      #     StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
+      #   elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000)
+      #     StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options)
+      #   else
+      #     StorageManager::SFTP.new(*all_server_hosts, ssh_options: ssh_options)
+      #   end
+      # end
     end
 
-    def build_large_file_url(post)
+    # The method to use for backing up image files.
-      "/data/sample/#{post.file_path_prefix}#{Danbooru.config.large_image_prefix}#{post.md5}.#{post.large_file_ext}"
+    def backup_storage_manager
+      # Don't perform any backups.
+      StorageManager::Null.new
+
+      # Backup files to /mnt/backup on the local filesystem.
+      # StorageManager::Local.new(base_dir: "/mnt/backup", hierarchical: false)
+
+      # Backup files to /mnt/backup on a remote system. Configure SSH settings
+      # in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
+      # StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {})
+
+      # Backup files to an S3 bucket. The bucket must already exist and be
+      # writable by you. Configure your S3 settings in aws_region and
+      # aws_credentials below, or in the s3_options param (ref:
+      # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method)
+      # StorageManager::S3.new("my_s3_bucket_name", s3_options: {})
     end
 
 #TAG CONFIGURATION
@@ -611,6 +643,14 @@ module Danbooru
     end
 
     # AWS config options
+    def aws_region
+      "us-east-1"
+    end
+
+    def aws_credentials
+      Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
+    end
+
     def aws_access_key_id
     end
 

test/factories/upload.rb

@@ -1,5 +1,3 @@
-require 'fileutils'
-
 FactoryGirl.define do
   factory(:upload) do
     rating "s"
@@ -15,51 +13,10 @@ FactoryGirl.define do
     end
 
     factory(:jpg_upload) do
-      content_type "image/jpeg"
       file do
         f = Tempfile.new
-        f.write(File.read("#{Rails.root}/test/files/test.jpg"))
+        IO.copy_stream("#{Rails.root}/test/files/test.jpg", f.path)
-        f.seek(0)
+        ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "test.jpg")
-        f
-      end
-    end
-
-    factory(:exif_jpg_upload) do
-      content_type "image/jpeg"
-      file_path do
-        FileUtils.cp("#{Rails.root}/test/files/test-exif-small.jpg", "#{Rails.root}/tmp")
-        "#{Rails.root}/tmp/test-exif-small.jpg"
-      end
-    end
-
-    factory(:blank_jpg_upload) do
-      content_type "image/jpeg"
-      file_path do
-        FileUtils.cp("#{Rails.root}/test/files/test-blank.jpg", "#{Rails.root}/tmp")
-        "#{Rails.root}/tmp/test-blank.jpg"
-      end
-    end
-
-    factory(:large_jpg_upload) do
-      file_ext "jpg"
-      content_type "image/jpeg"
-      file_path do
-        FileUtils.cp("#{Rails.root}/test/files/test-large.jpg", "#{Rails.root}/tmp")
-        "#{Rails.root}/tmp/test-large.jpg"
-      end
-    end
-
-    factory(:png_upload) do
-      file_path do
-        FileUtils.cp("#{Rails.root}/test/files/test.png", "#{Rails.root}/tmp")
-        "#{Rails.root}/tmp/test.png"
-      end
-    end
-
-    factory(:gif_upload) do
-      file_path do
-        FileUtils.cp("#{Rails.root}/test/files/test.gif", "#{Rails.root}/tmp")
-        "#{Rails.root}/tmp/test.gif"
       end
     end
   end

test/functional/post_replacements_controller_test.rb

@@ -39,7 +39,7 @@ class PostReplacementsControllerTest < ActionController::TestCase
         assert_response :success
         assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source)
         assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5)
-        assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file_path).hexdigest)
+        assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file(:original)).hexdigest)
       end
     end
 

test/test_helper.rb

@@ -38,9 +38,14 @@ class ActiveSupport::TestCase
     mock_missed_search_service!
     WebMock.allow_net_connect!
     Danbooru.config.stubs(:enable_sock_puppet_validation?).returns(false)
+
+    storage_manager = StorageManager::Local.new(base_dir: "#{Rails.root}/public/data/test")
+    Danbooru.config.stubs(:storage_manager).returns(storage_manager)
+    Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
   end
 
   teardown do
+    FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir)
     Cache.clear
   end
 end

test/test_helpers/download_helper.rb

@@ -1,18 +1,14 @@
 module DownloadTestHelper
   def assert_downloaded(expected_filesize, source)
-    tempfile = Tempfile.new("danbooru-test")
-    download = Downloads::File.new(source, tempfile.path)
-
     assert_nothing_raised(Downloads::File::Error) do
-      download.download!
+      tempfile = Downloads::File.new(source).download!
+      assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
+      tempfile.close!
     end
-
-    assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
   end
 
   def assert_rewritten(expected_source, test_source)
-    tempfile = Tempfile.new("danbooru-test")
+    download = Downloads::File.new(test_source)
-    download = Downloads::File.new(test_source, tempfile.path)
 
     rewritten_source, _, _ = download.before_download(test_source, {})
     assert_match(expected_source, rewritten_source, "Tested source URL: #{test_source}")

test/test_helpers/iqdb_test_helper.rb

@@ -23,7 +23,7 @@ module IqdbTestHelper
   end
 
   def mock_iqdb_matches!(post_or_source, matches)
-    source = post_or_source.is_a?(Post) ? post_or_source.complete_preview_file_url : post_or_source
+    source = post_or_source.is_a?(Post) ? post_or_source.preview_file_url : post_or_source
     url = "http://localhost:3004/similar?key=hunter2&url=#{CGI.escape source}&ref"
     body = matches.map { |post| { post_id: post.id } }.to_json
 

test/test_helpers/upload_test_helper.rb

@@ -1,23 +1,10 @@
 module UploadTestHelper
-  def upload_file(path, content_type, filename)
+  def upload_file(path)
-    tempfile = Tempfile.new(filename)
+    file = Tempfile.new(binmode: true)
-    FileUtils.copy_file(path, tempfile.path)
+    IO.copy_stream("#{Rails.root}/#{path}", file.path)
+    uploaded_file = ActionDispatch::Http::UploadedFile.new(tempfile: file, filename: File.basename(path))
 
-    (class << tempfile; self; end).class_eval do
+    yield uploaded_file if block_given?
-      alias local_path path
+    uploaded_file
-      define_method(:tempfile) {self}
-      define_method(:original_filename) {filename}
-      define_method(:content_type) {content_type}
-    end
-
-    tempfile
-  end
-
-  def upload_jpeg(path)
-    upload_file(path, "image/jpeg", File.basename(path))
-  end
-
-  def upload_zip(path)
-    upload_file(path, "application/zip", File.basename(path))
   end
 end

test/unit/downloads/art_station_test.rb

@@ -5,9 +5,7 @@ module Downloads
     context "a download for a (small) artstation image" do
       setup do
         @source = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
-        @tempfile = Tempfile.new("danbooru-test")
+        @download = Downloads::File.new(@source)
-        @download = Downloads::File.new(@source, @tempfile.path)
-        @download.download!
       end
 
       should "download the large image instead" do
@@ -18,8 +16,7 @@ module Downloads
     context "for an image where an original does not exist" do
       setup do
         @source = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg"
-        @tempfile = Tempfile.new("danbooru-test")
+        @download = Downloads::File.new(@source)
-        @download = Downloads::File.new(@source, @tempfile.path)
         @download.download!
       end
 
@@ -38,8 +35,7 @@ module Downloads
     context "a download for a https://$artist.artstation.com/projects/$id page" do
       setup do
         @source = "https://dantewontdie.artstation.com/projects/YZK5q"
-        @tempfile = Tempfile.new("danbooru-test")
+        @download = Downloads::File.new(@source)
-        @download = Downloads::File.new(@source, @tempfile.path)
         @download.download!
       end
 

test/unit/downloads/deviant_art_test.rb

@@ -5,9 +5,8 @@ module Downloads
     context "a download for a deviant art html page" do
       setup do
         @source = "http://starbitt.deviantart.com/art/09271X-636962118"
-        @tempfile = Tempfile.new("danbooru-test")
+        @download = Downloads::File.new(@source)
-        @download = Downloads::File.new(@source, @tempfile.path)
+        @tempfile = @download.download!
-        @download.download!
       end
 
       should "set the html page as the source" do

test/unit/downloads/file_test.rb

@@ -5,12 +5,7 @@ module Downloads
     context "A twitter video download" do
       setup do
         @source = "https://twitter.com/CincinnatiZoo/status/859073537713328129"
-        @tempfile = Tempfile.new("danbooru-test")
+        @download = Downloads::File.new(@source)
-        @download = Downloads::File.new(@source, @tempfile.path)
-      end
-
-      teardown do
-        @tempfile.close
       end
 
       should "preserve the twitter source" do
@@ -22,12 +17,8 @@ module Downloads
     context "A post download" do
       setup do
         @source = "http://www.google.com/intl/en_ALL/images/logo.gif"
+        @download = Downloads::File.new(@source)
         @tempfile = Tempfile.new("danbooru-test")
-        @download = Downloads::File.new(@source, @tempfile.path)
-      end
-
-      teardown do
-        @tempfile.close
       end
 
       context "that fails" do
@@ -49,10 +40,9 @@ module Downloads
       end
 
       should "store the file in the tempfile path" do
-        @download.download!
+        tempfile = @download.download!
         assert_equal(@source, @download.source)
-        assert(::File.exists?(@tempfile.path), "temp file should exist")
+        assert_operator(tempfile.size, :>, 0, "should have data")
-        assert(::File.size(@tempfile.path) > 0, "should have data")
       end
     end
   end

test/unit/downloads/pixiv_test.rb

@@ -4,13 +4,9 @@ module Downloads
   class PixivTest < ActiveSupport::TestCase
     context "An ugoira site for pixiv" do
       setup do
-        @tempfile = Tempfile.new("danbooru-test")
+        @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
-        @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364", @tempfile.path)
+        @tempfile = @download.download!
-        @download.download!
+        @tempfile.close!
-      end
-
-      teardown do
-        @tempfile.unlink
       end
 
       should "capture the data" do

test/unit/pixiv_ugoira_converter_test.rb

@@ -3,9 +3,7 @@ require "test_helper"
 class PixivUgoiraConverterTest < ActiveSupport::TestCase
   context "An ugoira converter" do
     setup do
-      @zipped_body = "#{Rails.root}/test/fixtures/ugoira.zip"
+      @zipfile = upload_file("test/fixtures/ugoira.zip").tempfile
-      @write_file = Tempfile.new("converted")
-      @preview_write_file = Tempfile.new("preview")
       @frame_data = [
         {"file" => "000000.jpg", "delay" => 200},
         {"file" => "000001.jpg", "delay" => 200},
@@ -15,16 +13,11 @@ class PixivUgoiraConverterTest < ActiveSupport::TestCase
       ]
     end
 
-    teardown do
-      @write_file.unlink
-      @preview_write_file.unlink
-    end
-
     should "output to webm" do
-      @converter = PixivUgoiraConverter
+      sample_file = PixivUgoiraConverter.generate_webm(@zipfile, @frame_data)
-      @converter.convert(@zipped_body, @write_file.path, @preview_write_file.path, @frame_data)
+      preview_file = PixivUgoiraConverter.generate_preview(@zipfile)
-      assert_operator(File.size(@write_file.path), :>, 1_000)
+      assert_operator(sample_file.size, :>, 1_000)
-      assert_operator(File.size(@preview_write_file.path), :>, 0)
+      assert_operator(preview_file.size, :>, 0)
     end
   end
-end
+end

test/unit/post_replacement_test.rb

@@ -1,15 +1,6 @@
 require 'test_helper'
 
 class PostReplacementTest < ActiveSupport::TestCase
-  def upload_file(path, filename, &block)
-    Tempfile.open do |file|
-      file.write(File.read(path))
-      file.seek(0)
-      uploaded_file = ActionDispatch::Http::UploadedFile.new(tempfile: file, filename: filename)
-      yield uploaded_file
-    end
-  end
-
   def setup
     super
 
@@ -68,7 +59,7 @@ class PostReplacementTest < ActiveSupport::TestCase
           assert_equal(5969, @post.file_size)
           assert_equal("png", @post.file_ext)
           assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", @post.md5)
-          assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file_path).hexdigest)
+          assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file).hexdigest)
           assert_equal("https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png", @post.source)
         end
       end
@@ -102,7 +93,7 @@ class PostReplacementTest < ActiveSupport::TestCase
         assert_equal(8558, @post.file_size)
         assert_equal("gif", @post.file_ext)
         assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5)
-        assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file_path).hexdigest)
+        assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file).hexdigest)
         assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source)
       end
 
@@ -155,18 +146,17 @@ class PostReplacementTest < ActiveSupport::TestCase
         assert_equal(16275, @post.file_size)
         assert_equal("png", @post.file_ext)
         assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
-        assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file_path).hexdigest)
+        assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
         assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
         assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
       end
 
-      should "delete the old files after three days" do
+      should "delete the old files after thirty days" do
-        old_file_path, old_preview_file_path, old_large_file_path = @post.file_path, @post.preview_file_path, @post.large_file_path
+        old_file_path, old_preview_file_path = @post.file(:original).path, @post.file(:preview).path
         @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
 
         assert(File.exists?(old_file_path))
         assert(File.exists?(old_preview_file_path))
-        assert(File.exists?(old_large_file_path))
 
         Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
           Delayed::Worker.new.work_off
@@ -174,7 +164,6 @@ class PostReplacementTest < ActiveSupport::TestCase
 
         assert_not(File.exists?(old_file_path))
         assert_not(File.exists?(old_preview_file_path))
-        assert_not(File.exists?(old_large_file_path))
       end
     end
 
@@ -188,7 +177,7 @@ class PostReplacementTest < ActiveSupport::TestCase
         assert_equal(2804, @post.file_size)
         assert_equal("zip", @post.file_ext)
         assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5)
-        assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file_path).hexdigest)
+        assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest)
 
         assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source)
         assert_equal([{"file"=>"000000.jpg", "delay"=>125}, {"file"=>"000001.jpg", "delay"=>125}], @post.pixiv_ugoira_frame_data.data)
@@ -201,17 +190,15 @@ class PostReplacementTest < ActiveSupport::TestCase
         @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
         @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
 
-        assert(File.exists?(@post.file_path))
+        assert_nothing_raised { @post.file(:original) }
-        assert(File.exists?(@post.preview_file_path))
+        assert_nothing_raised { @post.file(:preview) }
-        assert(File.exists?(@post.large_file_path))
 
         Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
           Delayed::Worker.new.work_off
         end
 
-        assert(File.exists?(@post.file_path))
+        assert_nothing_raised { @post.file(:original) }
-        assert(File.exists?(@post.preview_file_path))
+        assert_nothing_raised { @post.file(:preview) }
-        assert(File.exists?(@post.large_file_path))
       end
     end
 
@@ -231,14 +218,14 @@ class PostReplacementTest < ActiveSupport::TestCase
           Delayed::Worker.new.work_off
         end
 
-        assert(File.exists?(@post1.file_path))
+        assert_nothing_raised { @post1.file(:original) }
-        assert(File.exists?(@post2.file_path))
+        assert_nothing_raised { @post2.file(:original) }
       end
     end
 
     context "a post with an uploaded file" do
       should "work" do
-        upload_file("#{Rails.root}/test/files/test.png", "test.png") do |file|
+        upload_file("test/files/test.png") do |file|
           @post.replace!(replacement_file: file, replacement_url: "")
           assert_equal(@post.md5, Digest::MD5.file(file.tempfile).hexdigest)
           assert_equal("file://test.png", @post.replacements.last.replacement_url)
@@ -268,7 +255,7 @@ class PostReplacementTest < ActiveSupport::TestCase
 
     context "a post with the same file" do
       should "not raise a duplicate error" do
-        upload_file("#{Rails.root}/test/files/test.jpg", "test.jpg") do |file|
+        upload_file("test/files/test.jpg") do |file|
           assert_nothing_raised do
             @post.replace!(replacement_file: file, replacement_url: "")
           end
@@ -276,7 +263,7 @@ class PostReplacementTest < ActiveSupport::TestCase
       end
 
       should "not queue a deletion or log a comment" do
-        upload_file("#{Rails.root}/test/files/test.jpg", "test.jpg") do |file|
+        upload_file("test/files/test.jpg") do |file|
           assert_no_difference(["@post.comments.count"]) do
             @post.replace!(replacement_file: file, replacement_url: "")
           end

test/unit/post_test.rb

@@ -35,17 +35,15 @@ class PostTest < ActiveSupport::TestCase
       end
 
       should "delete the files" do
-        assert_equal(true, File.exists?(@post.preview_file_path))
+        assert_nothing_raised { @post.file(:preview) }
-        assert_equal(true, File.exists?(@post.large_file_path))
+        assert_nothing_raised { @post.file(:original) }
-        assert_equal(true, File.exists?(@post.file_path))
 
         TestAfterCommit.with_commits(true) do
           @post.expunge!
         end
 
-        assert_equal(false, File.exists?(@post.preview_file_path))
+        assert_raise(StandardError) { @post.file(:preview) }
-        assert_equal(false, File.exists?(@post.large_file_path))
+        assert_raise(StandardError) { @post.file(:original) }
-        assert_equal(false, File.exists?(@post.file_path))
       end
 
       should "remove all favorites" do

test/unit/storage_manager_test.rb

@@ -0,0 +1,124 @@
+require 'test_helper'
+
+class StorageManagerTest < ActiveSupport::TestCase
+  BASE_DIR = "#{Rails.root}/tmp/test-storage"
+
+  setup do
+    CurrentUser.ip_addr = "127.0.0.1"
+  end
+
+  context "StorageManager::Local" do
+    setup do
+      @storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data")
+    end
+
+    teardown do
+      FileUtils.rm_rf(BASE_DIR)
+    end
+
+    context "#store method" do
+      should "store the file" do
+        @storage_manager.store(StringIO.new("data"), "#{BASE_DIR}/test.txt")
+
+        assert("data", File.read("#{BASE_DIR}/test.txt"))
+      end
+
+      should "overwrite the file if it already exists" do
+        @storage_manager.store(StringIO.new("foo"), "#{BASE_DIR}/test.txt")
+        @storage_manager.store(StringIO.new("bar"), "#{BASE_DIR}/test.txt")
+
+        assert("bar", File.read("#{BASE_DIR}/test.txt"))
+      end
+    end
+
+    context "#delete method" do
+      should "delete the file" do
+        @storage_manager.store(StringIO.new("data"), "test.txt")
+        @storage_manager.delete("test.txt")
+
+        assert_not(File.exist?("#{BASE_DIR}/test.txt"))
+      end
+
+      should "not fail if the file doesn't exist" do
+        assert_nothing_raised { @storage_manager.delete("dne.txt") }
+      end
+    end
+
+    context "#store_file and #delete_file methods" do
+      setup do
+        @post = FactoryGirl.create(:post, file_ext: "png")
+
+        @storage_manager.store_file(StringIO.new("data"), @post, :preview)
+        @storage_manager.store_file(StringIO.new("data"), @post, :large)
+        @storage_manager.store_file(StringIO.new("data"), @post, :original)
+
+        @file_path = "#{BASE_DIR}/preview/#{@post.md5}.jpg"
+        @large_file_path = "#{BASE_DIR}/sample/sample-#{@post.md5}.jpg"
+        @preview_file_path = "#{BASE_DIR}/#{@post.md5}.#{@post.file_ext}"
+      end
+
+      should "store the files at the correct path" do
+        assert(File.exist?(@file_path))
+        assert(File.exist?(@large_file_path))
+        assert(File.exist?(@preview_file_path))
+      end
+
+      should "delete the files" do
+        @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :preview)
+        @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :large)
+        @storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :original)
+
+        assert_not(File.exist?(@file_path))
+        assert_not(File.exist?(@large_file_path))
+        assert_not(File.exist?(@preview_file_path))
+      end
+    end
+
+    context "#file_url method" do
+      should "return the correct urls" do
+        @post = FactoryGirl.create(:post, file_ext: "png")
+        @storage_manager.stubs(:tagged_filenames).returns(false)
+
+        assert_equal("/data/#{@post.md5}.png", @storage_manager.file_url(@post, :original))
+        assert_equal("/data/sample/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large))
+        assert_equal("/data/preview/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview))
+      end
+    end
+  end
+
+  context "StorageManager::Hybrid" do
+    setup do
+      @post1 = FactoryGirl.build(:post, id: 1, file_ext: "png")
+      @post2 = FactoryGirl.build(:post, id: 2, file_ext: "png")
+
+      @storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type|
+        if id.odd?
+          StorageManager::Local.new(base_dir: "#{BASE_DIR}/i1", base_url: "/i1")
+        else
+          StorageManager::Local.new(base_dir: "#{BASE_DIR}/i2", base_url: "/i2")
+        end
+      end
+    end
+
+    teardown do
+      FileUtils.rm_rf(BASE_DIR)
+    end
+
+    context "#store_file method" do
+      should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do
+        @storage_manager.store_file(StringIO.new("post1"), @post1, :original)
+        @storage_manager.store_file(StringIO.new("post2"), @post2, :original)
+
+        assert(File.exist?("#{BASE_DIR}/i1/#{@post1.md5}.png"))
+        assert(File.exist?("#{BASE_DIR}/i2/#{@post2.md5}.png"))
+      end
+    end
+
+    context "#file_url method" do
+      should "generate /i1 urls for odd posts and /i2 urls for even posts" do
+        assert_equal("/i1/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
+        assert_equal("/i2/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
+      end
+    end
+  end
+end

test/unit/upload_test.rb

@@ -17,21 +17,15 @@ class UploadTest < ActiveSupport::TestCase
     teardown do
       CurrentUser.user = nil
       CurrentUser.ip_addr = nil
-
-      @upload.delete_temp_file if @upload
     end
 
     context "An upload" do
-      teardown do
-        FileUtils.rm_f(Dir.glob("#{Rails.root}/tmp/test.*"))
-      end
-
       context "from a user that is limited" do
         setup do
           CurrentUser.user = FactoryGirl.create(:user, :created_at => 1.year.ago)
           User.any_instance.stubs(:upload_limit).returns(0)
         end
-        
+
         should "fail creation" do
           @upload = FactoryGirl.build(:jpg_upload, :tag_string => "")
           @upload.save
@@ -41,73 +35,51 @@ class UploadTest < ActiveSupport::TestCase
 
       context "image size calculator" do
         should "discover the dimensions for a compressed SWF" do
-          @upload = FactoryGirl.create(:upload, :file_path => "#{Rails.root}/test/files/compressed.swf")
+          @upload = FactoryGirl.create(:upload, file: upload_file("test/files/compressed.swf"))
-          @upload.calculate_dimensions(@upload.file_path)
+          assert_equal([607, 756], @upload.calculate_dimensions)
-          assert_equal(607, @upload.image_width)
-          assert_equal(756, @upload.image_height)
         end
 
         should "discover the dimensions for a JPG with JFIF data" do
           @upload = FactoryGirl.create(:jpg_upload)
-          assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
+          assert_equal([500, 335], @upload.calculate_dimensions)
-          assert_equal(500, @upload.image_width)
-          assert_equal(335, @upload.image_height)
         end
 
         should "discover the dimensions for a JPG with EXIF data" do
-          @upload = FactoryGirl.create(:exif_jpg_upload)
+          @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test-exif-small.jpg"))
-          assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
+          assert_equal([529, 600], @upload.calculate_dimensions)
-          assert_equal(529, @upload.image_width)
-          assert_equal(600, @upload.image_height)
         end
 
         should "discover the dimensions for a JPG with no header data" do
-          @upload = FactoryGirl.create(:blank_jpg_upload)
+          @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test-blank.jpg"))
-          assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
+          assert_equal([668, 996], @upload.calculate_dimensions)
-          assert_equal(668, @upload.image_width)
-          assert_equal(996, @upload.image_height)
         end
 
         should "discover the dimensions for a PNG" do
-          @upload = FactoryGirl.create(:png_upload)
+          @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.png"))
-          assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
+          assert_equal([768, 1024], @upload.calculate_dimensions)
-          assert_equal(768, @upload.image_width)
-          assert_equal(1024, @upload.image_height)
         end
 
         should "discover the dimensions for a GIF" do
-          @upload = FactoryGirl.create(:gif_upload)
+          @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.gif"))
-          assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
+          assert_equal([400, 400], @upload.calculate_dimensions)
-          assert_equal(400, @upload.image_width)
-          assert_equal(400, @upload.image_height)
         end
       end
 
       context "content type calculator" do
         should "know how to parse jpeg, png, gif, and swf file headers" do
-          @upload = FactoryGirl.create(:jpg_upload)
+          @upload = FactoryGirl.build(:jpg_upload)
-          assert_equal("image/jpeg", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.jpg"))
+          assert_equal("jpg", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.jpg")))
-          assert_equal("image/gif", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.gif"))
+          assert_equal("gif", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.gif")))
-          assert_equal("image/png", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.png"))
+          assert_equal("png", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.png")))
-          assert_equal("application/x-shockwave-flash", @upload.file_header_to_content_type("#{Rails.root}/test/files/compressed.swf"))
+          assert_equal("swf", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/compressed.swf")))
-          assert_equal("application/octet-stream", @upload.file_header_to_content_type("#{Rails.root}/README.md"))
+          assert_equal("bin", @upload.file_header_to_file_ext(File.open("#{Rails.root}/README.md")))
-        end
-
-        should "know how to parse jpeg, png, gif, and swf content types" do
-          @upload = FactoryGirl.create(:jpg_upload)
-          assert_equal("jpg", @upload.content_type_to_file_ext("image/jpeg"))
-          assert_equal("gif", @upload.content_type_to_file_ext("image/gif"))
-          assert_equal("png", @upload.content_type_to_file_ext("image/png"))
-          assert_equal("swf", @upload.content_type_to_file_ext("application/x-shockwave-flash"))
-          assert_equal("bin", @upload.content_type_to_file_ext(""))
         end
       end
 
       context "downloader" do
         context "for a zip that is not an ugoira" do
           should "not validate" do
-            FileUtils.cp("#{Rails.root}/test/files/invalid_ugoira.zip", "#{Rails.root}/tmp")
+            @upload = FactoryGirl.create(:upload, file: upload_file("test/files/invalid_ugoira.zip"))
-            @upload = Upload.create(:file => upload_zip("#{Rails.root}/tmp/invalid_ugoira.zip"), :rating => "q", :tag_string => "xxx")
             @upload.process!
             assert_equal("error: RuntimeError - missing frame data for ugoira", @upload.status)
           end
@@ -116,30 +88,15 @@ class UploadTest < ActiveSupport::TestCase
         context "that is a pixiv ugoira" do
           setup do
             @url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654"
-            @upload = FactoryGirl.create(:source_upload, :source => @url, :tag_string => "ugoira")
+            @upload = FactoryGirl.create(:upload, :source => @url, :tag_string => "ugoira")
-            @output_file = Tempfile.new("download")
           end
 
-          teardown do
-            @output_file.unlink
-          end
-          
           should "process successfully" do
-            @upload.download_from_source(@output_file.path)
+            _, _, output_file = @upload.download_from_source(@url, "")
-            assert_operator(File.size(@output_file.path), :>, 1_000)
+            assert_operator(output_file.size, :>, 1_000)
-            assert_equal("application/zip", @upload.file_header_to_content_type(@output_file.path))
+            assert_equal("zip", @upload.file_header_to_file_ext(output_file))
-            assert_equal("zip", @upload.content_type_to_file_ext(@upload.file_header_to_content_type(@output_file.path)))
           end
         end
-
-        should "initialize the final path after downloading a file" do
-          @upload = FactoryGirl.create(:source_upload)
-          path = "#{Rails.root}/tmp/test.download.jpg"
-          assert_nothing_raised {@upload.download_from_source(path)}
-          assert(File.exists?(path))
-          assert_equal(8558, File.size(path))
-          assert_equal(path, @upload.file_path)
-        end
       end
 
       context "determining if a file is downloadable" do
@@ -161,46 +118,32 @@ class UploadTest < ActiveSupport::TestCase
 
       context "file processor" do
         should "parse and process a cgi file representation" do
-          FileUtils.cp("#{Rails.root}/test/files/test.jpg", "#{Rails.root}/tmp")
+          @upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.jpg"))
-          @upload = Upload.new(:file => upload_jpeg("#{Rails.root}/tmp/test.jpg"))
+          assert_nothing_raised {@upload.process_upload}
-          assert_nothing_raised {@upload.convert_cgi_file}
+          assert_equal(28086, @upload.file_size)
-          assert(File.exists?(@upload.file_path))
-          assert_equal(28086, File.size(@upload.file_path))
         end
 
         should "process a transparent png" do
-          FileUtils.cp("#{Rails.root}/test/files/alpha.png", "#{Rails.root}/tmp")
+          @upload = FactoryGirl.create(:upload, file: upload_file("test/files/alpha.png"))
-          @upload = Upload.new(:file => upload_file("#{Rails.root}/tmp/alpha.png", "image/png", "alpha.png"))
+          assert_nothing_raised {@upload.process_upload}
-          assert_nothing_raised {@upload.convert_cgi_file}
+          assert_equal(1136, @upload.file_size)
-          assert(File.exists?(@upload.file_path))
-          assert_equal(1136, File.size(@upload.file_path))
         end
       end
 
       context "hash calculator" do
         should "caculate the hash" do
           @upload = FactoryGirl.create(:jpg_upload)
-          @upload.calculate_hash(@upload.file_path)
+          @upload.process_upload
           assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
         end
       end
 
       context "resizer" do
-        teardown do
-          FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/preview/test.*.jpg"))
-          FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/sample/test.*.jpg"))
-          FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/test.*.jpg"))
-        end
-
         should "generate several resized versions of the image" do
-          @upload = FactoryGirl.create(:large_jpg_upload)
+          @upload = FactoryGirl.create(:upload, file_ext: "jpg", image_width: 1356, image_height: 911, file: upload_file("test/files/test-large.jpg"))
-          @upload.calculate_hash(@upload.file_path)
+          preview_file, sample_file = @upload.generate_resizes
-          @upload.calculate_dimensions(@upload.file_path)
+          assert_operator(preview_file.size, :>, 1_000)
-          assert_nothing_raised {@upload.generate_resizes(@upload.file_path)}
+          assert_operator(sample_file.size, :>, 1_000)
-          assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.small_image_width)))
-          assert(File.size(@upload.resized_file_path_for(Danbooru.config.small_image_width)) > 0)
-          assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.large_image_width)))
-          assert(File.size(@upload.resized_file_path_for(Danbooru.config.large_image_width)) > 0)
         end
       end
 
@@ -215,13 +158,10 @@ class UploadTest < ActiveSupport::TestCase
       context "with an artist commentary" do
         setup do
           @upload = FactoryGirl.create(:source_upload,
-            :rating => "s",
+            include_artist_commentary: "1",
-            :uploader_ip_addr => "127.0.0.1",
+            artist_commentary_title: "",
-            :tag_string => "hoge foo"
+            artist_commentary_desc: "blah",
-            )
+          )
-          @upload.include_artist_commentary = "1"
-          @upload.artist_commentary_title = ""
-          @upload.artist_commentary_desc = "blah"
         end
 
         should "create an artist commentary when processed" do
@@ -255,15 +195,17 @@ class UploadTest < ActiveSupport::TestCase
         assert_equal(post.id, @upload.post_id)
         assert_equal("completed", @upload.status)
       end
+
+      context "automatic tagging" do
+        should "tag animated png files" do
+          @upload = FactoryGirl.build(:upload, file_ext: "png", file: upload_file("test/files/apng/normal_apng.png"))
+          assert_equal("animated_png", @upload.automatic_tags)
+        end
+      end
     end
 
     should "process completely for a pixiv ugoira" do
-      @upload = FactoryGirl.create(:source_upload,
+      @upload = FactoryGirl.create(:source_upload, source: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654")
-        :source => "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654",
-        :rating => "s",
-        :uploader_ip_addr => "127.0.0.1",
-        :tag_string => "hoge foo"
-        )
       assert_difference(["PixivUgoiraFrameData.count", "Post.count"]) do
         @upload.process!
         assert_equal([], @upload.errors.full_messages)
@@ -274,18 +216,18 @@ class UploadTest < ActiveSupport::TestCase
       assert_equal(60, post.image_width)
       assert_equal(60, post.image_height)
       assert_equal("https://i.pximg.net/img-zip-ugoira/img/2014/10/05/23/42/23/46378654_ugoira1920x1080.zip", post.source)
-      assert_operator(File.size(post.large_file_path), :>, 0)
+      assert_nothing_raised { post.file(:original) }
-      assert_operator(File.size(post.preview_file_path), :>, 0)
+      assert_nothing_raised { post.file(:large) }
+      assert_nothing_raised { post.file(:preview) }
     end
 
     should "process completely for an uploaded image" do
       @upload = FactoryGirl.create(:jpg_upload,
         :rating => "s",
         :uploader_ip_addr => "127.0.0.1",
-        :tag_string => "hoge foo"
+        :tag_string => "hoge foo",
+        :file => upload_file("test/files/test.jpg"),
         )
-      @upload.file = upload_jpeg("#{Rails.root}/test/files/test.jpg")
-      @upload.convert_cgi_file
 
       assert_difference("Post.count") do
         assert_nothing_raised {@upload.process!}
@@ -297,8 +239,8 @@ class UploadTest < ActiveSupport::TestCase
       assert_equal("127.0.0.1", post.uploader_ip_addr.to_s)
       assert_equal(@upload.md5, post.md5)
       assert_equal("jpg", post.file_ext)
-      assert(File.exists?(post.file_path))
+      assert_nothing_raised { post.file(:original) }
-      assert_equal(28086, File.size(post.file_path))
+      assert_equal(28086, post.file(:original).size)
       assert_equal(post.id, @upload.post_id)
       assert_equal("completed", @upload.status)
     end
@@ -310,16 +252,5 @@ class UploadTest < ActiveSupport::TestCase
         assert_nothing_raised {@upload.process!}
       end
     end
-
-    should "delete the temporary file upon completion" do
-      @upload = FactoryGirl.create(:source_upload,
-        :rating => "s",
-        :uploader_ip_addr => "127.0.0.1",
-        :tag_string => "hoge foo"
-        )
-
-      @upload.process!
-      assert(!File.exists?(@upload.temp_file_path))
-    end
   end
 end