From 8b85bbe8eaeea3e61483a12fac8bd13c5f04dfd4 Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 3 Sep 2021 22:37:19 -0500 Subject: [PATCH] storage manager: remove 'hybrid' and 'match' manager. Remove StorageManager::Hybrid and StorageManager::Match. These were used to store uploads on different servers based on the post ID or file sample type. This is no longer used in production because in hindsight it's a lot more difficult to manage uploads when they're fragmented across different servers. If you need this, you can do tricks with network filesystems to get the same effect. For example, if you want to store some files on server A and others on server B, then mount servers A and B as network filesystems (with e.g. sshfs, Samba, NFS, etc), and use symlinks to point subdirectories at either server A or B. --- app/logical/storage_manager/hybrid.rb | 23 ------ app/logical/storage_manager/match.rb | 109 -------------------------- config/danbooru_default_config.rb | 15 ---- test/unit/storage_manager_test.rb | 68 ---------------- 4 files changed, 215 deletions(-) delete mode 100644 app/logical/storage_manager/hybrid.rb delete mode 100644 app/logical/storage_manager/match.rb diff --git a/app/logical/storage_manager/hybrid.rb b/app/logical/storage_manager/hybrid.rb deleted file mode 100644 index 9b105f09e..000000000 --- a/app/logical/storage_manager/hybrid.rb +++ /dev/null @@ -1,23 +0,0 @@ -class StorageManager::Hybrid < StorageManager - attr_reader :submanager - - def initialize(&block) - @submanager = block - end - - def store_file(io, post, type) - submanager[post.id, post.md5, post.file_ext, type].store_file(io, post, type) - end - - def delete_file(post_id, md5, file_ext, type) - submanager[post_id, md5, file_ext, type].delete_file(post_id, md5, file_ext, type) - end - - def open_file(post, type) - submanager[post.id, post.md5, post.file_ext, type].open_file(post, type) - end - - def file_url(post, type, **options) - submanager[post.id, post.md5, post.file_ext, type].file_url(post, type, **options) - end -end diff --git a/app/logical/storage_manager/match.rb b/app/logical/storage_manager/match.rb deleted file mode 100644 index 84bb9a78a..000000000 --- a/app/logical/storage_manager/match.rb +++ /dev/null @@ -1,109 +0,0 @@ -# -# Generalizes the hybrid storage manager to be more declarative in -# syntax. Matches are executed in order of appearance so the first -# matching manager is returned. You should always add at least one -# manager with no constraints as a default case. -# -### Example -# -# StorageManager::Match.new do |matcher| -# matcher.add_manager(type: :crop) do -# StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", base_dir: "/var/www/raikou3") -# end -# -# matcher.add_manager(id: 1..850_000) do -# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/var/www/raikou1") -# end -# -# matcher.add_manager(id: 850_001..2_000_000) do -# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/var/www/raikou2") -# end -# -# matcher.add_manager(id: 1..3_000_000, type: [:large, :original]) do -# StorageManager::SFTP.new(*Danbooru.config.all_server_hosts, base_url: "https://hijiribe.donmai.us/data") -# end -# -# matcher.add_manager({}) do -# StorageManager::SFTP.new(*Danbooru.config.all_server_hosts, base_url: "#{CurrentUser.root_url}/data") -# end -# end -# - -class StorageManager::Match < StorageManager - def initialize - @managers = [] - - yield self if block_given? - end - - def add_manager(constraints) - manager = yield - @managers << [constraints, manager] - end - - def find(params) - @managers.each do |constraints, manager| - match = true - - if params[:id] && constraints[:id] && !constraints[:id].include?(params[:id].to_i) - match = false - end - - if constraints[:id] && !params[:id] - match = false - end - - if params[:type] && constraints[:type] - if constraints[:type].respond_to?(:include?) - if !constraints[:type].include?(params[:type]) - match = false - end - elsif constraints[:type] != params[:type] - match = false - end - end - - if constraints[:type] && !params[:type] - match = false - end - - if match - if block_given? - return yield(manager) - else - return manager - end - end - end - end - - def store_file(io, post, type) - find(id: post.id, type: type) do |manager| - manager.store_file(io, post, type) - end - end - - def delete_file(post_id, md5, file_ext, type) - find(id: post_id, type: type) do |manager| - manager.delete_file(post_id, md5, file_ext, type) - end - end - - def open_file(post, type) - find(id: post.id, type: type) do |manager| - manager.open_file(post, type) - end - end - - def file_url(post, type, **options) - find(id: post.id, type: type) do |manager| - manager.file_url(post, type, **options) - end - end - - def file_path(post, file_ext, type, **options) - find(id: post.id, type: type) do |manager| - manager.file_path(post, file_ext, type, **options) - end - end -end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 4ea40fabd..d9d891d7d 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -200,21 +200,6 @@ module Danbooru # Store files on one or more remote host(s). Configure SSH settings in # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", ssh_options: {}) - - # Select the storage method based on the post's id and type (preview, large, or original). - # StorageManager::Hybrid.new do |id, md5, file_ext, type| - # ssh_options = { user: "danbooru" } - # - # if type.in?([:large, :original]) && id.in?(0..850_000) - # StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options) - # elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000) - # StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options) - # elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000) - # StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options) - # else - # StorageManager::SFTP.new(*all_server_hosts, ssh_options: ssh_options) - # end - # end end # The method to use for backing up image files. diff --git a/test/unit/storage_manager_test.rb b/test/unit/storage_manager_test.rb index 15fb69c35..778a711bb 100644 --- a/test/unit/storage_manager_test.rb +++ b/test/unit/storage_manager_test.rb @@ -5,42 +5,6 @@ class StorageManagerTest < ActiveSupport::TestCase CurrentUser.ip_addr = "127.0.0.1" end - context "StorageManager::Match" do - setup do - @storage_manager = StorageManager::Match.new do |matcher| - matcher.add_manager(type: :crop) do - "crop" - end - - matcher.add_manager(type: [:large, :original]) do - "large or original" - end - - matcher.add_manager(id: 1..100) do - "first" - end - - matcher.add_manager(id: 101..200, type: :preview) do - "preview" - end - - matcher.add_manager({}) do - "default" - end - end - end - - should "find the different matches" do - assert_equal("large or original", @storage_manager.find(type: :original)) - assert_equal("crop", @storage_manager.find(type: :crop)) - assert_equal("large or original", @storage_manager.find(type: :large)) - assert_equal("preview", @storage_manager.find(type: :preview, id: 150)) - assert_equal("default", @storage_manager.find(type: :preview, id: 1000)) - assert_equal("crop", @storage_manager.find(type: :crop, id: 1_000)) - assert_equal("large or original", @storage_manager.find(type: :large, id: 1_000)) - end - end - context "StorageManager::Local" do setup do @storage_manager = StorageManager::Local.new(base_dir: @temp_dir, base_url: "/data") @@ -127,36 +91,4 @@ class StorageManagerTest < ActiveSupport::TestCase end end end - - context "StorageManager::Hybrid" do - setup do - @post1 = FactoryBot.build(:post, id: 1, file_ext: "png") - @post2 = FactoryBot.build(:post, id: 2, file_ext: "png") - - @storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type| - if id.odd? - StorageManager::Local.new(base_dir: "#{@temp_dir}/i1", base_url: "/i1") - else - StorageManager::Local.new(base_dir: "#{@temp_dir}/i2", base_url: "/i2") - end - end - end - - context "#store_file method" do - should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do - @storage_manager.store_file(StringIO.new("post1"), @post1, :original) - @storage_manager.store_file(StringIO.new("post2"), @post2, :original) - - assert(File.exist?("#{@temp_dir}/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png")) - assert(File.exist?("#{@temp_dir}/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png")) - end - end - - context "#file_url method" do - should "generate /i1 urls for odd posts and /i2 urls for even posts" do - assert_equal("/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original)) - assert_equal("/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original)) - end - end - end end