Remove StorageManager::SFTP.

Remove the SFTP file storage backend. Downstream users can use either
sshfs (which is what Danbooru now uses in production) or rclone instead.
The Ruby SFTP gem was much slower than sshfs.
This commit is contained in:
evazion
2021-12-01 23:46:20 -06:00
parent 5bf4e31c25
commit 6fc0854b4c
6 changed files with 1 additions and 98 deletions

View File

@@ -9,7 +9,6 @@ gem "delayed_job_active_record"
gem "simple_form"
gem "sanitize"
gem 'ruby-vips'
gem 'net-sftp'
gem 'diff-lcs', :require => "diff/lcs/array"
gem 'bcrypt', :require => "bcrypt"
gem 'rubyzip', :require => "zip"

View File

@@ -277,9 +277,6 @@ GEM
multi_json (1.15.0)
multi_xml (0.6.0)
multipart-post (2.1.1)
net-sftp (3.0.0)
net-ssh (>= 5.0.0, < 7.0.0)
net-ssh (6.1.0)
newrelic_rpm (8.0.0)
nio4r (2.5.8)
nokogiri (1.12.5)
@@ -525,7 +522,6 @@ DEPENDENCIES
minitest-reporters
mocha
mock_redis
net-sftp
newrelic_rpm
nokogiri
oauth2

View File

@@ -9,7 +9,6 @@
# @see StorageManager::Local
# @see StorageManager::Mirror
# @see StorageManager::Rclone
# @see StorageManager::SFTP
class StorageManager
class Error < StandardError; end

View File

@@ -1,81 +0,0 @@
# A StorageManager that stores files on a remote filesystem using SFTP.
class StorageManager::SFTP < StorageManager
DEFAULT_PERMISSIONS = 0o644
# http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start
DEFAULT_SSH_OPTIONS = {
timeout: 10,
logger: Rails.logger,
verbose: :fatal,
non_interactive: true
}
attr_reader :hosts, :ssh_options, :base_dir
def initialize(*hosts, base_dir: nil, ssh_options: {}, **options)
@hosts = hosts
@base_dir = base_dir.to_s
@ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options)
super(**options)
end
def store(file, dest_path)
dest_path = full_path(dest_path)
temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak"
each_host do |_host, sftp|
sftp.upload!(file.path, temp_upload_path)
sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS)
# `rename!` can't overwrite existing files, so if a file already exists
# at dest_path we move it out of the way first.
force { sftp.rename!(dest_path, dest_backup_path) }
force { sftp.rename!(temp_upload_path, dest_path) }
rescue StandardError => e
# if anything fails, try to move the original file back in place (if it was moved).
force { sftp.rename!(dest_backup_path, dest_path) }
raise Error, e
ensure
force { sftp.remove!(temp_upload_path) }
force { sftp.remove!(dest_backup_path) }
end
end
def delete(dest_path)
each_host do |_host, sftp|
force { sftp.remove!(full_path(dest_path)) }
end
end
def open(dest_path)
file = Tempfile.new(binmode: true)
Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp|
sftp.download!(full_path(dest_path), file.path)
end
file
end
protected
# Ignore "no such file" exceptions for the given operation.
def force
yield
rescue Net::SFTP::StatusException => e
raise Error, e unless e.description == "no such file"
end
def each_host
hosts.each do |host|
Net::SFTP.start(host, nil, ssh_options) do |sftp|
yield host, sftp
end
end
end
def full_path(path)
File.join(base_dir, path)
end
end

View File

@@ -196,10 +196,6 @@ module Danbooru
# base_dir - where to store files (default: under public/data)
# base_url - where to serve files from (default: https://#{hostname}/data)
StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data"))
# Store files on one or more remote host(s). Configure SSH settings in
# ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
# StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", ssh_options: {})
end
# The method to use for backing up image files.
@@ -209,10 +205,6 @@ module Danbooru
# Backup files to /mnt/backup on the local filesystem.
# StorageManager::Local.new(base_dir: "/mnt/backup")
# Backup files to /mnt/backup on a remote system. Configure SSH settings
# in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
# StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {})
end
# Any custom code you want to insert into the default layout without

View File

@@ -1,5 +1,3 @@
Rails.autoloaders.each do |autoloader|
autoloader.inflector.inflect(
"sftp" => "SFTP"
)
autoloader.inflector.inflect({})
end