jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

storage manager: remove 'hybrid' and 'match' manager.

Browse Source 
Remove StorageManager::Hybrid and StorageManager::Match. These were used
to store uploads on different servers based on the post ID or file
sample type. This is no longer used in production because in hindsight
it's a lot more difficult to manage uploads when they're fragmented
across different servers.

If you need this, you can do tricks with network filesystems to get the
same effect. For example, if you want to store some files on server A
and others on server B, then mount servers A and B as network
filesystems (with e.g. sshfs, Samba, NFS, etc), and use symlinks to
point subdirectories at either server A or B.
This commit is contained in:
evazion
2021-09-03 22:37:19 -05:00
parent 440d2c807f
commit 8b85bbe8ea
4 changed files with 0 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/logical/storage_manager/hybrid.rb
View File

@@ -1,23 +0,0 @@
class StorageManager::Hybrid < StorageManager
attr_reader :submanager
def initialize(&block)
@submanager = block
end
def store_file(io, post, type)
submanager[post.id, post.md5, post.file_ext, type].store_file(io, post, type)
end
def delete_file(post_id, md5, file_ext, type)
submanager[post_id, md5, file_ext, type].delete_file(post_id, md5, file_ext, type)
end
def open_file(post, type)
submanager[post.id, post.md5, post.file_ext, type].open_file(post, type)
end
def file_url(post, type, **options)
submanager[post.id, post.md5, post.file_ext, type].file_url(post, type, **options)
end
end

109
app/logical/storage_manager/match.rb
View File

@@ -1,109 +0,0 @@
#
# Generalizes the hybrid storage manager to be more declarative in
# syntax. Matches are executed in order of appearance so the first
# matching manager is returned. You should always add at least one
# manager with no constraints as a default case.
#
### Example
#
# StorageManager::Match.new do |matcher|
# matcher.add_manager(type: :crop) do
# StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", base_dir: "/var/www/raikou3")
# end
#
# matcher.add_manager(id: 1..850_000) do
# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/var/www/raikou1")
# end
#
# matcher.add_manager(id: 850_001..2_000_000) do
# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/var/www/raikou2")
# end
#
# matcher.add_manager(id: 1..3_000_000, type: [:large, :original]) do
# StorageManager::SFTP.new(*Danbooru.config.all_server_hosts, base_url: "https://hijiribe.donmai.us/data")
# end
#
# matcher.add_manager({}) do
# StorageManager::SFTP.new(*Danbooru.config.all_server_hosts, base_url: "#{CurrentUser.root_url}/data")
# end
# end
#
class StorageManager::Match < StorageManager
def initialize
@managers = []
yield self if block_given?
end
def add_manager(constraints)
manager = yield
@managers << [constraints, manager]
end
def find(params)
@managers.each do |constraints, manager|
match = true
if params[:id] && constraints[:id] && !constraints[:id].include?(params[:id].to_i)
match = false
end
if constraints[:id] && !params[:id]
match = false
end
if params[:type] && constraints[:type]
if constraints[:type].respond_to?(:include?)
if !constraints[:type].include?(params[:type])
match = false
end
elsif constraints[:type] != params[:type]
match = false
end
end
if constraints[:type] && !params[:type]
match = false
end
if match
if block_given?
return yield(manager)
else
return manager
end
end
end
end
def store_file(io, post, type)
find(id: post.id, type: type) do |manager|
manager.store_file(io, post, type)
end
end
def delete_file(post_id, md5, file_ext, type)
find(id: post_id, type: type) do |manager|
manager.delete_file(post_id, md5, file_ext, type)
end
end
def open_file(post, type)
find(id: post.id, type: type) do |manager|
manager.open_file(post, type)
end
end
def file_url(post, type, **options)
find(id: post.id, type: type) do |manager|
manager.file_url(post, type, **options)
end
end
def file_path(post, file_ext, type, **options)
find(id: post.id, type: type) do |manager|
manager.file_path(post, file_ext, type, **options)
end
end
end

15
config/danbooru_default_config.rb
View File

@@ -200,21 +200,6 @@ module Danbooru
# Store files on one or more remote host(s). Configure SSH settings in
# ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
# StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", ssh_options: {})
# Select the storage method based on the post's id and type (preview, large, or original).
# StorageManager::Hybrid.new do |id, md5, file_ext, type|
# ssh_options = { user: "danbooru" }
#
# if type.in?([:large, :original]) && id.in?(0..850_000)
# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options)
# elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000)
# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options)
# elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000)
# StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options)
# else
# StorageManager::SFTP.new(*all_server_hosts, ssh_options: ssh_options)
# end
# end
end
# The method to use for backing up image files.

68
test/unit/storage_manager_test.rb
View File

@@ -5,42 +5,6 @@ class StorageManagerTest < ActiveSupport::TestCase
CurrentUser.ip_addr = "127.0.0.1"
end
context "StorageManager::Match" do
setup do
@storage_manager = StorageManager::Match.new do |matcher|
matcher.add_manager(type: :crop) do
"crop"
end
matcher.add_manager(type: [:large, :original]) do
"large or original"
end
matcher.add_manager(id: 1..100) do
"first"
end
matcher.add_manager(id: 101..200, type: :preview) do
"preview"
end
matcher.add_manager({}) do
"default"
end
end
end
should "find the different matches" do
assert_equal("large or original", @storage_manager.find(type: :original))
assert_equal("crop", @storage_manager.find(type: :crop))
assert_equal("large or original", @storage_manager.find(type: :large))
assert_equal("preview", @storage_manager.find(type: :preview, id: 150))
assert_equal("default", @storage_manager.find(type: :preview, id: 1000))
assert_equal("crop", @storage_manager.find(type: :crop, id: 1_000))
assert_equal("large or original", @storage_manager.find(type: :large, id: 1_000))
end
end
context "StorageManager::Local" do
setup do
@storage_manager = StorageManager::Local.new(base_dir: @temp_dir, base_url: "/data")
@@ -127,36 +91,4 @@ class StorageManagerTest < ActiveSupport::TestCase
end
end
end
context "StorageManager::Hybrid" do
setup do
@post1 = FactoryBot.build(:post, id: 1, file_ext: "png")
@post2 = FactoryBot.build(:post, id: 2, file_ext: "png")
@storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type|
if id.odd?
StorageManager::Local.new(base_dir: "#{@temp_dir}/i1", base_url: "/i1")
else
StorageManager::Local.new(base_dir: "#{@temp_dir}/i2", base_url: "/i2")
end
end
end
context "#store_file method" do
should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do
@storage_manager.store_file(StringIO.new("post1"), @post1, :original)
@storage_manager.store_file(StringIO.new("post2"), @post2, :original)
assert(File.exist?("#{@temp_dir}/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png"))
assert(File.exist?("#{@temp_dir}/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png"))
end
end
context "#file_url method" do
should "generate /i1 urls for odd posts and /i2 urls for even posts" do
assert_equal("/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
assert_equal("/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
end
end
end
end