storage manager: refactor base_dir option.

Fix it so the `base_dir` option is only required by subclasses that
actually use it. The StorageManager::Mirror class doesn't use it.
This commit is contained in:
evazion
2021-10-27 20:13:36 -05:00
parent 4095d14f2a
commit f593828bb9
6 changed files with 46 additions and 19 deletions

View File

@@ -3,22 +3,25 @@
# are handled by a StorageManager.
#
# A StorageManager has methods for saving, deleting, and opening files, and for
# generates URLs for images.
# generating URLs for files.
#
# @abstract
# @see StorageManager::Local
# @see StorageManager::Mirror
# @see StorageManager::Rclone
# @see StorageManager::SFTP
class StorageManager
class Error < StandardError; end
attr_reader :base_url, :base_dir
attr_reader :base_url
# Initialize a storage manager object.
# @param base_url [String] the base URL where images are stored (ex: "https://cdn.donmai.us/")
# @param base_dir [String] the base directory where images are stored (ex: "/var/www/danbooru/public/images")
def initialize(base_url: nil, base_dir: nil)
@base_url = base_url.to_s.chomp("/")
@base_dir = base_dir.to_s
#
# @param base_url [String, nil] the base URL where files are served from (ex:
# "https://cdn.donmai.us"), or nil if the files don't have an URL (they're
# stored in a publicly inaccessible location).
def initialize(base_url: nil)
@base_url = base_url
end
# Store the given file at the given path. If a file already exists at that
@@ -41,16 +44,16 @@ class StorageManager
# Return a readonly copy of the file located at the given path.
# @param path [String] the remote path of the file to open
# @return [MediaFile] the image file
# @return [File] the file
def open(path)
raise NotImplementedError, "open not implemented"
end
# Return the full URL of the file at the given path, or nil if the file
# doesn't have an URL.
# @return [String, nil] the file URL
def file_url(path)
return nil if base_dir.nil?
File.join(base_url, path)
end
def full_path(path)
File.join(base_dir, path)
end
end

View File

@@ -2,6 +2,14 @@
class StorageManager::Local < StorageManager
DEFAULT_PERMISSIONS = 0o644
attr_reader :base_dir
# @param base_url [String] the base directory where files are stored (ex: "/home/danbooru/public/data")
def initialize(base_dir: nil, **options)
@base_dir = base_dir.to_s
super(**options)
end
def store(io, dest_path)
temp_path = full_path(dest_path) + "-" + SecureRandom.uuid + ".tmp"
@@ -24,4 +32,10 @@ class StorageManager::Local < StorageManager
def open(path)
File.open(full_path(path), "r", binmode: true)
end
protected
def full_path(path)
File.join(base_dir, path)
end
end

View File

@@ -1,8 +1,8 @@
# A null StorageManager that doesn't store files at all. Used for testing or
# disabling backups.
class StorageManager::Null < StorageManager
def initialize(base_url: "/", base_dir: "/")
super
def initialize
super(base_url: nil)
end
def store(io, path)

View File

@@ -5,11 +5,12 @@
# @see https://rclone.org/
class StorageManager::Rclone < StorageManager
class Error < StandardError; end
attr_reader :remote, :bucket, :rclone_path, :rclone_options
attr_reader :remote, :bucket, :rclone_path, :rclone_options, :base_dir
def initialize(remote:, bucket:, rclone_path: "rclone", rclone_options: {}, **options)
def initialize(remote:, bucket:, base_dir: nil, rclone_path: "rclone", rclone_options: {}, **options)
@remote = remote
@bucket = bucket
@base_dir = base_dir.to_s
@rclone_path = rclone_path
@rclone_options = rclone_options
super(**options)
@@ -37,4 +38,8 @@ class StorageManager::Rclone < StorageManager
def key(path)
":#{remote}:#{bucket}#{full_path(path)}"
end
def full_path(path)
File.join(base_dir, path)
end
end

View File

@@ -10,10 +10,11 @@ class StorageManager::SFTP < StorageManager
non_interactive: true
}
attr_reader :hosts, :ssh_options
attr_reader :hosts, :ssh_options, :base_dir
def initialize(*hosts, ssh_options: {}, **options)
def initialize(*hosts, base_dir: nil, ssh_options: {}, **options)
@hosts = hosts
@base_dir = base_dir.to_s
@ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options)
super(**options)
end
@@ -73,4 +74,8 @@ class StorageManager::SFTP < StorageManager
end
end
end
def full_path(path)
File.join(base_dir, path)
end
end

View File

@@ -51,7 +51,7 @@ class ActiveSupport::TestCase
@temp_dir = Dir.mktmpdir("danbooru-temp-")
storage_manager = StorageManager::Local.new(base_url: "https://www.example.com/data", base_dir: @temp_dir)
Danbooru.config.stubs(:storage_manager).returns(storage_manager)
Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new(base_url: "/", base_dir: "/"))
Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
end
teardown do