Remove StorageManager::SFTP.
Remove the SFTP file storage backend. Downstream users can use either sshfs (which is what Danbooru now uses in production) or rclone instead. The Ruby SFTP gem was much slower than sshfs.
This commit is contained in:
1
Gemfile
1
Gemfile
@@ -9,7 +9,6 @@ gem "delayed_job_active_record"
|
||||
gem "simple_form"
|
||||
gem "sanitize"
|
||||
gem 'ruby-vips'
|
||||
gem 'net-sftp'
|
||||
gem 'diff-lcs', :require => "diff/lcs/array"
|
||||
gem 'bcrypt', :require => "bcrypt"
|
||||
gem 'rubyzip', :require => "zip"
|
||||
|
||||
@@ -277,9 +277,6 @@ GEM
|
||||
multi_json (1.15.0)
|
||||
multi_xml (0.6.0)
|
||||
multipart-post (2.1.1)
|
||||
net-sftp (3.0.0)
|
||||
net-ssh (>= 5.0.0, < 7.0.0)
|
||||
net-ssh (6.1.0)
|
||||
newrelic_rpm (8.0.0)
|
||||
nio4r (2.5.8)
|
||||
nokogiri (1.12.5)
|
||||
@@ -525,7 +522,6 @@ DEPENDENCIES
|
||||
minitest-reporters
|
||||
mocha
|
||||
mock_redis
|
||||
net-sftp
|
||||
newrelic_rpm
|
||||
nokogiri
|
||||
oauth2
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
# @see StorageManager::Local
|
||||
# @see StorageManager::Mirror
|
||||
# @see StorageManager::Rclone
|
||||
# @see StorageManager::SFTP
|
||||
class StorageManager
|
||||
class Error < StandardError; end
|
||||
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# A StorageManager that stores files on a remote filesystem using SFTP.
|
||||
class StorageManager::SFTP < StorageManager
|
||||
DEFAULT_PERMISSIONS = 0o644
|
||||
|
||||
# http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start
|
||||
DEFAULT_SSH_OPTIONS = {
|
||||
timeout: 10,
|
||||
logger: Rails.logger,
|
||||
verbose: :fatal,
|
||||
non_interactive: true
|
||||
}
|
||||
|
||||
attr_reader :hosts, :ssh_options, :base_dir
|
||||
|
||||
def initialize(*hosts, base_dir: nil, ssh_options: {}, **options)
|
||||
@hosts = hosts
|
||||
@base_dir = base_dir.to_s
|
||||
@ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options)
|
||||
super(**options)
|
||||
end
|
||||
|
||||
def store(file, dest_path)
|
||||
dest_path = full_path(dest_path)
|
||||
temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
|
||||
dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak"
|
||||
|
||||
each_host do |_host, sftp|
|
||||
sftp.upload!(file.path, temp_upload_path)
|
||||
sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS)
|
||||
|
||||
# `rename!` can't overwrite existing files, so if a file already exists
|
||||
# at dest_path we move it out of the way first.
|
||||
force { sftp.rename!(dest_path, dest_backup_path) }
|
||||
force { sftp.rename!(temp_upload_path, dest_path) }
|
||||
rescue StandardError => e
|
||||
# if anything fails, try to move the original file back in place (if it was moved).
|
||||
force { sftp.rename!(dest_backup_path, dest_path) }
|
||||
raise Error, e
|
||||
ensure
|
||||
force { sftp.remove!(temp_upload_path) }
|
||||
force { sftp.remove!(dest_backup_path) }
|
||||
end
|
||||
end
|
||||
|
||||
def delete(dest_path)
|
||||
each_host do |_host, sftp|
|
||||
force { sftp.remove!(full_path(dest_path)) }
|
||||
end
|
||||
end
|
||||
|
||||
def open(dest_path)
|
||||
file = Tempfile.new(binmode: true)
|
||||
|
||||
Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp|
|
||||
sftp.download!(full_path(dest_path), file.path)
|
||||
end
|
||||
|
||||
file
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
# Ignore "no such file" exceptions for the given operation.
|
||||
def force
|
||||
yield
|
||||
rescue Net::SFTP::StatusException => e
|
||||
raise Error, e unless e.description == "no such file"
|
||||
end
|
||||
|
||||
def each_host
|
||||
hosts.each do |host|
|
||||
Net::SFTP.start(host, nil, ssh_options) do |sftp|
|
||||
yield host, sftp
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def full_path(path)
|
||||
File.join(base_dir, path)
|
||||
end
|
||||
end
|
||||
@@ -196,10 +196,6 @@ module Danbooru
|
||||
# base_dir - where to store files (default: under public/data)
|
||||
# base_url - where to serve files from (default: https://#{hostname}/data)
|
||||
StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data"))
|
||||
|
||||
# Store files on one or more remote host(s). Configure SSH settings in
|
||||
# ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
|
||||
# StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", ssh_options: {})
|
||||
end
|
||||
|
||||
# The method to use for backing up image files.
|
||||
@@ -209,10 +205,6 @@ module Danbooru
|
||||
|
||||
# Backup files to /mnt/backup on the local filesystem.
|
||||
# StorageManager::Local.new(base_dir: "/mnt/backup")
|
||||
|
||||
# Backup files to /mnt/backup on a remote system. Configure SSH settings
|
||||
# in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
|
||||
# StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {})
|
||||
end
|
||||
|
||||
# Any custom code you want to insert into the default layout without
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
Rails.autoloaders.each do |autoloader|
|
||||
autoloader.inflector.inflect(
|
||||
"sftp" => "SFTP"
|
||||
)
|
||||
autoloader.inflector.inflect({})
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user