From 29d2e7fed2357fd57552472c13defee9f348a741 Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 17 Mar 2021 00:01:01 -0500 Subject: [PATCH] storage manager: remove `hierarchical` option. Remove the `hierarchical` file storage option. This means that image files are always stored in MD5-based subdirectories, like this: https://danbooru.donmai.us/data/original/f3/a7/f3a70a89c350b5ed4db22dbb25b934bb.jpg https://danbooru.donmai.us/data/sample/f3/a7/sample-f3a70a89c350b5ed4db22dbb25b934bb.jpg https://danbooru.donmai.us/data/preview/f3/a7/f3a70a89c350b5ed4db22dbb25b934bb.jpg instead of in a single flat directory, like this: https://danbooru.donmai.us/data/original/f3a70a89c350b5ed4db22dbb25b934bb.jpg This option is removed because storing files in a single directory is a bad idea for large installations, and migrating from a single directory to subdirectories later is a pain. Downstream boorus who still have files in the old layout can migrate by running this script: `./script/fixes/077_symlink_subdirectories.rb` This will create symlinks that redirect the 00-ff subdirectories back to the current directory, so that you can still store files in a single directory, but use URLs containing subdirectories. You should also make sure to remove the `hierarchical` option from `storage_manager` in `config/danbooru_local_config.rb` if you set it there. --- app/logical/storage_manager.rb | 11 +++------ app/logical/storage_manager/match.rb | 6 ++--- config/danbooru_default_config.rb | 14 +++++------ script/fixes/077_symlink_subdirectories.rb | 27 ++++++++++++++++++++++ test/test_helper.rb | 4 ++-- test/unit/post_test.rb | 8 +++---- test/unit/storage_manager_test.rb | 22 ++++++++++-------- 7 files changed, 57 insertions(+), 35 deletions(-) create mode 100755 script/fixes/077_symlink_subdirectories.rb diff --git a/app/logical/storage_manager.rb b/app/logical/storage_manager.rb index d4dbc147a..95c7d3805 100644 --- a/app/logical/storage_manager.rb +++ b/app/logical/storage_manager.rb @@ -1,12 +1,11 @@ class StorageManager class Error < StandardError; end - attr_reader :base_url, :base_dir, :hierarchical, :tagged_filenames + attr_reader :base_url, :base_dir, :tagged_filenames - def initialize(base_url:, base_dir:, hierarchical: false, tagged_filenames: Danbooru.config.enable_seo_post_urls) + def initialize(base_url:, base_dir:, tagged_filenames: Danbooru.config.enable_seo_post_urls) @base_url = base_url.chomp("/") @base_dir = base_dir - @hierarchical = hierarchical @tagged_filenames = tagged_filenames end @@ -98,11 +97,7 @@ class StorageManager end def subdir_for(md5) - if hierarchical - "#{md5[0..1]}/#{md5[2..3]}/" - else - "" - end + "#{md5[0..1]}/#{md5[2..3]}/" end def seo_tags(post) diff --git a/app/logical/storage_manager/match.rb b/app/logical/storage_manager/match.rb index 4ed2eb647..84bb9a78a 100644 --- a/app/logical/storage_manager/match.rb +++ b/app/logical/storage_manager/match.rb @@ -8,15 +8,15 @@ # # StorageManager::Match.new do |matcher| # matcher.add_manager(type: :crop) do -# StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", hierarchical: true, base_dir: "/var/www/raikou3") +# StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", base_dir: "/var/www/raikou3") # end # # matcher.add_manager(id: 1..850_000) do -# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", hierarchical: true, base_dir: "/var/www/raikou1") +# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/var/www/raikou1") # end # # matcher.add_manager(id: 850_001..2_000_000) do -# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", hierarchical: true, base_dir: "/var/www/raikou2") +# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/var/www/raikou2") # end # # matcher.add_manager(id: 1..3_000_000, type: [:large, :original]) do diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 4d2011943..940595b2f 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -33,7 +33,7 @@ module Danbooru # you don't support HTTPS. Protip: use ngrok.com for easy HTTPS support # during development. def canonical_url - "https://#{Danborou.config.hostname}" + "https://#{Danbooru.config.hostname}" end # Contact email address of the admin. @@ -152,22 +152,20 @@ module Danbooru # Store files on the local filesystem. # base_dir - where to store files (default: under public/data) # base_url - where to serve files from (default: https://#{hostname}/data) - # hierarchical: false - store files in a single directory - # hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash - StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data"), hierarchical: false) + StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data")) # Store files on one or more remote host(s). Configure SSH settings in # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) - # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {}) + # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", ssh_options: {}) # Select the storage method based on the post's id and type (preview, large, or original). # StorageManager::Hybrid.new do |id, md5, file_ext, type| # ssh_options = { user: "danbooru" } # # if type.in?([:large, :original]) && id.in?(0..850_000) - # StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options) + # StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options) # elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000) - # StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options) + # StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options) # elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000) # StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options) # else @@ -182,7 +180,7 @@ module Danbooru StorageManager::Null.new # Backup files to /mnt/backup on the local filesystem. - # StorageManager::Local.new(base_dir: "/mnt/backup", hierarchical: false) + # StorageManager::Local.new(base_dir: "/mnt/backup") # Backup files to /mnt/backup on a remote system. Configure SSH settings # in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) diff --git a/script/fixes/077_symlink_subdirectories.rb b/script/fixes/077_symlink_subdirectories.rb new file mode 100755 index 000000000..992af869e --- /dev/null +++ b/script/fixes/077_symlink_subdirectories.rb @@ -0,0 +1,27 @@ +#!/usr/bin/env ruby + +require_relative "../../config/environment" + +def create_symlinks(dir) + FileUtils.mkdir_p(dir) + + (0..255).each do |i| + subdir = "#{dir}/#{"%.2x" % i}" + + if File.exist?(subdir) + puts "skipping #{subdir}" + else + puts "ln -sf . #{subdir}" + FileUtils.ln_sf(".", subdir) + end + end +end + +root = Rails.root.join("public/data") + +create_symlinks(root) +create_symlinks("#{root}/sample") +create_symlinks("#{root}/preview") +create_symlinks("#{root}/crop") + +FileUtils.ln_sf(".", "#{root}/original") unless File.exist?("#{root}/original") diff --git a/test/test_helper.rb b/test/test_helper.rb index 65462e850..85692e82e 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -49,9 +49,9 @@ class ActiveSupport::TestCase Socket.stubs(:gethostname).returns("www.example.com") @temp_dir = Dir.mktmpdir("danbooru-temp-") - storage_manager = StorageManager::Local.new(base_dir: @temp_dir) + storage_manager = StorageManager::Local.new(base_url: "https://www.example.com/data", base_dir: @temp_dir) Danbooru.config.stubs(:storage_manager).returns(storage_manager) - Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new) + Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new(base_url: "/", base_dir: "/")) end teardown do diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index dcb3d6088..361e9c03f 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -1972,14 +1972,14 @@ class PostTest < ActiveSupport::TestCase context "URLs:" do should "generate the correct urls for animated gifs" do - manager = StorageManager::Local.new(base_url: "https://test.com/data") + manager = StorageManager::Local.new(base_url: "https://test.com/data", base_dir: "/") Danbooru.config.stubs(:storage_manager).returns(manager) @post = build(:post, md5: "deadbeef", file_ext: "gif", tag_string: "animated_gif") - assert_equal("https://test.com/data/preview/deadbeef.jpg", @post.preview_file_url) - assert_equal("https://test.com/data/deadbeef.gif", @post.large_file_url) - assert_equal("https://test.com/data/deadbeef.gif", @post.file_url) + assert_equal("https://test.com/data/preview/de/ad/deadbeef.jpg", @post.preview_file_url) + assert_equal("https://test.com/data/original/de/ad/deadbeef.gif", @post.large_file_url) + assert_equal("https://test.com/data/original/de/ad/deadbeef.gif", @post.file_url) end end diff --git a/test/unit/storage_manager_test.rb b/test/unit/storage_manager_test.rb index 9f9812ad3..15fb69c35 100644 --- a/test/unit/storage_manager_test.rb +++ b/test/unit/storage_manager_test.rb @@ -81,10 +81,11 @@ class StorageManagerTest < ActiveSupport::TestCase @storage_manager.store_file(StringIO.new("data"), @post, :preview) @storage_manager.store_file(StringIO.new("data"), @post, :large) @storage_manager.store_file(StringIO.new("data"), @post, :original) + subdir = "#{@post.md5[0..1]}/#{@post.md5[2..3]}" - @file_path = "#{@temp_dir}/preview/#{@post.md5}.jpg" - @large_file_path = "#{@temp_dir}/sample/sample-#{@post.md5}.jpg" - @preview_file_path = "#{@temp_dir}/#{@post.md5}.#{@post.file_ext}" + @file_path = "#{@temp_dir}/preview/#{subdir}/#{@post.md5}.jpg" + @large_file_path = "#{@temp_dir}/sample/#{subdir}/sample-#{@post.md5}.jpg" + @preview_file_path = "#{@temp_dir}/original/#{subdir}/#{@post.md5}.#{@post.file_ext}" end should "store the files at the correct path" do @@ -108,10 +109,11 @@ class StorageManagerTest < ActiveSupport::TestCase should "return the correct urls" do @post = FactoryBot.create(:post, file_ext: "png") @storage_manager.stubs(:tagged_filenames).returns(false) + subdir = "#{@post.md5[0..1]}/#{@post.md5[2..3]}" - assert_equal("/data/#{@post.md5}.png", @storage_manager.file_url(@post, :original)) - assert_equal("/data/sample/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large)) - assert_equal("/data/preview/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview)) + assert_equal("/data/original/#{subdir}/#{@post.md5}.png", @storage_manager.file_url(@post, :original)) + assert_equal("/data/sample/#{subdir}/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large)) + assert_equal("/data/preview/#{subdir}/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview)) end should "return the correct url for flash files" do @@ -145,15 +147,15 @@ class StorageManagerTest < ActiveSupport::TestCase @storage_manager.store_file(StringIO.new("post1"), @post1, :original) @storage_manager.store_file(StringIO.new("post2"), @post2, :original) - assert(File.exist?("#{@temp_dir}/i1/#{@post1.md5}.png")) - assert(File.exist?("#{@temp_dir}/i2/#{@post2.md5}.png")) + assert(File.exist?("#{@temp_dir}/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png")) + assert(File.exist?("#{@temp_dir}/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png")) end end context "#file_url method" do should "generate /i1 urls for odd posts and /i2 urls for even posts" do - assert_equal("/i1/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original)) - assert_equal("/i2/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original)) + assert_equal("/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original)) + assert_equal("/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original)) end end end