From 6fc0854b4c441e7170e3189f5693f51909ef47e9 Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 1 Dec 2021 23:46:20 -0600 Subject: [PATCH] Remove StorageManager::SFTP. Remove the SFTP file storage backend. Downstream users can use either sshfs (which is what Danbooru now uses in production) or rclone instead. The Ruby SFTP gem was much slower than sshfs. --- Gemfile | 1 - Gemfile.lock | 4 -- app/logical/storage_manager.rb | 1 - app/logical/storage_manager/sftp.rb | 81 ----------------------------- config/danbooru_default_config.rb | 8 --- config/initializers/zeitwerk.rb | 4 +- 6 files changed, 1 insertion(+), 98 deletions(-) delete mode 100644 app/logical/storage_manager/sftp.rb diff --git a/Gemfile b/Gemfile index aa357dadb..9a305d6d6 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,6 @@ gem "delayed_job_active_record" gem "simple_form" gem "sanitize" gem 'ruby-vips' -gem 'net-sftp' gem 'diff-lcs', :require => "diff/lcs/array" gem 'bcrypt', :require => "bcrypt" gem 'rubyzip', :require => "zip" diff --git a/Gemfile.lock b/Gemfile.lock index f178445ff..3bcc7df2d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -277,9 +277,6 @@ GEM multi_json (1.15.0) multi_xml (0.6.0) multipart-post (2.1.1) - net-sftp (3.0.0) - net-ssh (>= 5.0.0, < 7.0.0) - net-ssh (6.1.0) newrelic_rpm (8.0.0) nio4r (2.5.8) nokogiri (1.12.5) @@ -525,7 +522,6 @@ DEPENDENCIES minitest-reporters mocha mock_redis - net-sftp newrelic_rpm nokogiri oauth2 diff --git a/app/logical/storage_manager.rb b/app/logical/storage_manager.rb index 94a957bc4..8a2a8a4a0 100644 --- a/app/logical/storage_manager.rb +++ b/app/logical/storage_manager.rb @@ -9,7 +9,6 @@ # @see StorageManager::Local # @see StorageManager::Mirror # @see StorageManager::Rclone -# @see StorageManager::SFTP class StorageManager class Error < StandardError; end diff --git a/app/logical/storage_manager/sftp.rb b/app/logical/storage_manager/sftp.rb deleted file mode 100644 index bb10aa329..000000000 --- a/app/logical/storage_manager/sftp.rb +++ /dev/null @@ -1,81 +0,0 @@ -# A StorageManager that stores files on a remote filesystem using SFTP. -class StorageManager::SFTP < StorageManager - DEFAULT_PERMISSIONS = 0o644 - - # http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start - DEFAULT_SSH_OPTIONS = { - timeout: 10, - logger: Rails.logger, - verbose: :fatal, - non_interactive: true - } - - attr_reader :hosts, :ssh_options, :base_dir - - def initialize(*hosts, base_dir: nil, ssh_options: {}, **options) - @hosts = hosts - @base_dir = base_dir.to_s - @ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options) - super(**options) - end - - def store(file, dest_path) - dest_path = full_path(dest_path) - temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp" - dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak" - - each_host do |_host, sftp| - sftp.upload!(file.path, temp_upload_path) - sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS) - - # `rename!` can't overwrite existing files, so if a file already exists - # at dest_path we move it out of the way first. - force { sftp.rename!(dest_path, dest_backup_path) } - force { sftp.rename!(temp_upload_path, dest_path) } - rescue StandardError => e - # if anything fails, try to move the original file back in place (if it was moved). - force { sftp.rename!(dest_backup_path, dest_path) } - raise Error, e - ensure - force { sftp.remove!(temp_upload_path) } - force { sftp.remove!(dest_backup_path) } - end - end - - def delete(dest_path) - each_host do |_host, sftp| - force { sftp.remove!(full_path(dest_path)) } - end - end - - def open(dest_path) - file = Tempfile.new(binmode: true) - - Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp| - sftp.download!(full_path(dest_path), file.path) - end - - file - end - - protected - - # Ignore "no such file" exceptions for the given operation. - def force - yield - rescue Net::SFTP::StatusException => e - raise Error, e unless e.description == "no such file" - end - - def each_host - hosts.each do |host| - Net::SFTP.start(host, nil, ssh_options) do |sftp| - yield host, sftp - end - end - end - - def full_path(path) - File.join(base_dir, path) - end -end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 8be2f0852..f5768ff15 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -196,10 +196,6 @@ module Danbooru # base_dir - where to store files (default: under public/data) # base_url - where to serve files from (default: https://#{hostname}/data) StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data")) - - # Store files on one or more remote host(s). Configure SSH settings in - # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) - # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", ssh_options: {}) end # The method to use for backing up image files. @@ -209,10 +205,6 @@ module Danbooru # Backup files to /mnt/backup on the local filesystem. # StorageManager::Local.new(base_dir: "/mnt/backup") - - # Backup files to /mnt/backup on a remote system. Configure SSH settings - # in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) - # StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {}) end # Any custom code you want to insert into the default layout without diff --git a/config/initializers/zeitwerk.rb b/config/initializers/zeitwerk.rb index 7e5b9951d..538a761fe 100644 --- a/config/initializers/zeitwerk.rb +++ b/config/initializers/zeitwerk.rb @@ -1,5 +1,3 @@ Rails.autoloaders.each do |autoloader| - autoloader.inflector.inflect( - "sftp" => "SFTP" - ) + autoloader.inflector.inflect({}) end