Add storage managers (local, sftp, s3, hybrid).

app/logical/storage_manager/hybrid.rb

@@ -0,0 +1,23 @@
+class StorageManager::Hybrid < StorageManager
+  attr_reader :submanager
+
+  def initialize(&block)
+    @submanager = block
+  end
+
+  def store_file(io, post, type)
+    submanager[post.id, post.md5, post.file_ext, type].store_file(io, post, type)
+  end
+
+  def delete_file(post_id, md5, file_ext, type)
+    submanager[post_id, md5, file_ext, type].delete_file(post_id, md5, file_ext, type)
+  end
+
+  def open_file(io, post, type)
+    submanager[post.id, post.md5, post.file_ext, type].open_file(post, type)
+  end
+
+  def file_url(post, type)
+    submanager[post.id, post.md5, post.file_ext, type].file_url(post, type)
+  end
+end

app/logical/storage_manager/local.rb

@@ -0,0 +1,25 @@
+class StorageManager::Local < StorageManager
+  DEFAULT_PERMISSIONS = 0644
+
+  def store(io, dest_path)
+    temp_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
+
+    FileUtils.mkdir_p(File.dirname(temp_path))
+    bytes_copied = IO.copy_stream(io, temp_path)
+    raise Error, "store failed: #{bytes_copied}/#{io.size} bytes copied" if bytes_copied != io.size
+
+    FileUtils.chmod(DEFAULT_PERMISSIONS, temp_path)
+    File.rename(temp_path, dest_path)
+  rescue StandardError => e
+    FileUtils.rm_f(temp_path)
+    raise Error, e
+  end
+
+  def delete(path)
+    FileUtils.rm_f(path)
+  end
+
+  def open(path)
+    File.open(path, "r", binmode: true)
+  end
+end

app/logical/storage_manager/null.rb

@@ -0,0 +1,13 @@
+class StorageManager::Null < StorageManager
+  def store(io, path)
+    # no-op
+  end
+
+  def delete(path)
+    # no-op
+  end
+
+  def open(path)
+    # no-op
+  end
+end

app/logical/storage_manager/s3.rb

@@ -0,0 +1,43 @@
+class StorageManager::S3 < StorageManager
+  # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method
+  DEFAULT_S3_OPTIONS = {
+    region: Danbooru.config.aws_region,
+    credentials: Danbooru.config.aws_credentials,
+    logger: Rails.logger,
+  }
+
+  # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#put_object-instance_method
+  DEFAULT_PUT_OPTIONS = {
+    acl: "public-read",
+    storage_class: "STANDARD", # STANDARD, STANDARD_IA, REDUCED_REDUNDANCY
+    cache_control: "public, max-age=#{1.year.to_i}",
+    #content_type: "image/jpeg" # XXX should set content type
+  }
+
+  attr_reader :bucket, :client, :s3_options
+
+  def initialize(bucket, client: nil, s3_options: {}, **options)
+    @bucket = bucket
+    @s3_options = DEFAULT_S3_OPTIONS.merge(s3_options)
+    @client = client || Aws::S3::Client.new(**@s3_options)
+    super(**options)
+  end
+
+  def store(io, path)
+    data = io.read
+    base64_md5 = Digest::MD5.base64digest(data)
+    client.put_object(bucket: bucket, key: path, body: data, content_md5: base64_md5, **DEFAULT_PUT_OPTIONS)
+  end
+
+  def delete(path)
+    client.delete_object(bucket: bucket, key: path)
+  rescue Aws::S3::Errors::NoSuchKey
+    # ignore
+  end
+
+  def open(path)
+    file = Tempfile.new(binmode: true)
+    client.get_object(bucket: bucket: key: path, response_target: file)
+    file
+  end
+end

app/logical/storage_manager/sftp.rb

@@ -0,0 +1,76 @@
+class StorageManager::SFTP < StorageManager
+  DEFAULT_PERMISSIONS = 0644
+
+  # http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start
+  DEFAULT_SSH_OPTIONS = {
+    timeout: 10,
+    logger: Rails.logger,
+    verbose: :fatal,
+    non_interactive: true,
+  }
+
+  attr_reader :hosts, :ssh_options
+
+  def initialize(*hosts, ssh_options: {}, **options)
+    @hosts = hosts
+    @ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options)
+    super(**options)
+  end
+
+  def store(file, dest_path)
+    temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
+    dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak"
+
+    each_host do |host, sftp|
+      begin
+        sftp.upload!(file.path, temp_upload_path)
+        sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS)
+
+        # `rename!` can't overwrite existing files, so if a file already exists
+        # at dest_path we move it out of the way first.
+        force { sftp.rename!(dest_path, dest_backup_path) }
+        force { sftp.rename!(temp_upload_path, dest_path) }
+      rescue StandardError => e
+        # if anything fails, try to move the original file back in place (if it was moved).
+        force { sftp.rename!(dest_backup_path, dest_path) }
+        raise Error, e
+      ensure
+        force { sftp.remove!(temp_upload_path) }
+        force { sftp.remove!(dest_backup_path) }
+      end
+    end
+  end
+
+  def delete(dest_path)
+    each_host do |host, sftp|
+      force { sftp.remove!(dest_path) }
+    end
+  end
+
+  def open(dest_path)
+    file = Tempfile.new(binmode: true)
+
+    Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp|
+      sftp.download!(dest_path, file.path)
+    end
+
+    file
+  end
+
+  protected
+
+  # Ignore "no such file" exceptions for the given operation.
+  def force
+    yield
+  rescue Net::SFTP::StatusException => e
+    raise Error, e unless e.description == "no such file"
+  end
+
+  def each_host
+    hosts.each do |host|
+      Net::SFTP.start(host, nil, ssh_options) do |sftp|
+        yield host, sftp
+      end
+    end
+  end
+end