jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

storage manager: remove hierarchical option.

Browse Source 
Remove the `hierarchical` file storage option. This means that image
files are always stored in MD5-based subdirectories, like this:

   https://danbooru.donmai.us/data/original/f3/a7/f3a70a89c350b5ed4db22dbb25b934bb.jpg
   https://danbooru.donmai.us/data/sample/f3/a7/sample-f3a70a89c350b5ed4db22dbb25b934bb.jpg
   https://danbooru.donmai.us/data/preview/f3/a7/f3a70a89c350b5ed4db22dbb25b934bb.jpg

instead of in a single flat directory, like this:

   https://danbooru.donmai.us/data/original/f3a70a89c350b5ed4db22dbb25b934bb.jpg

This option is removed because storing files in a single directory is a
bad idea for large installations, and migrating from a single directory
to subdirectories later is a pain.

Downstream boorus who still have files in the old layout can migrate by
running this script:

   `./script/fixes/077_symlink_subdirectories.rb`

This will create symlinks that redirect the 00-ff subdirectories back to
the current directory, so that you can still store files in a single
directory, but use URLs containing subdirectories.

You should also make sure to remove the `hierarchical` option from
`storage_manager` in `config/danbooru_local_config.rb` if you set it
there.
This commit is contained in:
evazion
2021-03-17 00:01:01 -05:00
parent a620a71b59
commit 29d2e7fed2
7 changed files with 57 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/logical/storage_manager.rb
View File

@@ -1,12 +1,11 @@
class StorageManager
class Error < StandardError; end
attr_reader :base_url, :base_dir, :hierarchical, :tagged_filenames
attr_reader :base_url, :base_dir, :tagged_filenames
def initialize(base_url:, base_dir:, hierarchical: false, tagged_filenames: Danbooru.config.enable_seo_post_urls)
def initialize(base_url:, base_dir:, tagged_filenames: Danbooru.config.enable_seo_post_urls)
@base_url = base_url.chomp("/")
@base_dir = base_dir
@hierarchical = hierarchical
@tagged_filenames = tagged_filenames
end
@@ -98,11 +97,7 @@ class StorageManager
end
def subdir_for(md5)
if hierarchical
"#{md5[0..1]}/#{md5[2..3]}/"
else
""
end
"#{md5[0..1]}/#{md5[2..3]}/"
end
def seo_tags(post)

6
app/logical/storage_manager/match.rb
View File

@@ -8,15 +8,15 @@
#
# StorageManager::Match.new do |matcher|
# matcher.add_manager(type: :crop) do
# StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", hierarchical: true, base_dir: "/var/www/raikou3")
# StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", base_dir: "/var/www/raikou3")
# end
#
# matcher.add_manager(id: 1..850_000) do
# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", hierarchical: true, base_dir: "/var/www/raikou1")
# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/var/www/raikou1")
# end
#
# matcher.add_manager(id: 850_001..2_000_000) do
# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", hierarchical: true, base_dir: "/var/www/raikou2")
# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/var/www/raikou2")
# end
#
# matcher.add_manager(id: 1..3_000_000, type: [:large, :original]) do

14
config/danbooru_default_config.rb
View File

@@ -33,7 +33,7 @@ module Danbooru
# you don't support HTTPS. Protip: use ngrok.com for easy HTTPS support
# during development.
def canonical_url
"https://#{Danborou.config.hostname}"
"https://#{Danbooru.config.hostname}"
end
# Contact email address of the admin.
@@ -152,22 +152,20 @@ module Danbooru
# Store files on the local filesystem.
# base_dir - where to store files (default: under public/data)
# base_url - where to serve files from (default: https://#{hostname}/data)
# hierarchical: false - store files in a single directory
# hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash
StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data"), hierarchical: false)
StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data"))
# Store files on one or more remote host(s). Configure SSH settings in
# ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
# StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {})
# StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", ssh_options: {})
# Select the storage method based on the post's id and type (preview, large, or original).
# StorageManager::Hybrid.new do |id, md5, file_ext, type|
# ssh_options = { user: "danbooru" }
#
# if type.in?([:large, :original]) && id.in?(0..850_000)
# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options)
# elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000)
# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options)
# elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000)
# StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options)
# else
@@ -182,7 +180,7 @@ module Danbooru
StorageManager::Null.new
# Backup files to /mnt/backup on the local filesystem.
# StorageManager::Local.new(base_dir: "/mnt/backup", hierarchical: false)
# StorageManager::Local.new(base_dir: "/mnt/backup")
# Backup files to /mnt/backup on a remote system. Configure SSH settings
# in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)

27
script/fixes/077_symlink_subdirectories.rb Executable file
View File

@@ -0,0 +1,27 @@
#!/usr/bin/env ruby
require_relative "../../config/environment"
def create_symlinks(dir)
FileUtils.mkdir_p(dir)
(0..255).each do |i|
subdir = "#{dir}/#{"%.2x" % i}"
if File.exist?(subdir)
puts "skipping #{subdir}"
else
puts "ln -sf . #{subdir}"
FileUtils.ln_sf(".", subdir)
end
end
end
root = Rails.root.join("public/data")
create_symlinks(root)
create_symlinks("#{root}/sample")
create_symlinks("#{root}/preview")
create_symlinks("#{root}/crop")
FileUtils.ln_sf(".", "#{root}/original") unless File.exist?("#{root}/original")

4
test/test_helper.rb
View File

@@ -49,9 +49,9 @@ class ActiveSupport::TestCase
Socket.stubs(:gethostname).returns("www.example.com")
@temp_dir = Dir.mktmpdir("danbooru-temp-")
storage_manager = StorageManager::Local.new(base_dir: @temp_dir)
storage_manager = StorageManager::Local.new(base_url: "https://www.example.com/data", base_dir: @temp_dir)
Danbooru.config.stubs(:storage_manager).returns(storage_manager)
Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new(base_url: "/", base_dir: "/"))
end
teardown do

8
test/unit/post_test.rb
View File

@@ -1972,14 +1972,14 @@ class PostTest < ActiveSupport::TestCase
context "URLs:" do
should "generate the correct urls for animated gifs" do
manager = StorageManager::Local.new(base_url: "https://test.com/data")
manager = StorageManager::Local.new(base_url: "https://test.com/data", base_dir: "/")
Danbooru.config.stubs(:storage_manager).returns(manager)
@post = build(:post, md5: "deadbeef", file_ext: "gif", tag_string: "animated_gif")
assert_equal("https://test.com/data/preview/deadbeef.jpg", @post.preview_file_url)
assert_equal("https://test.com/data/deadbeef.gif", @post.large_file_url)
assert_equal("https://test.com/data/deadbeef.gif", @post.file_url)
assert_equal("https://test.com/data/preview/de/ad/deadbeef.jpg", @post.preview_file_url)
assert_equal("https://test.com/data/original/de/ad/deadbeef.gif", @post.large_file_url)
assert_equal("https://test.com/data/original/de/ad/deadbeef.gif", @post.file_url)
end
end

22
test/unit/storage_manager_test.rb
View File

@@ -81,10 +81,11 @@ class StorageManagerTest < ActiveSupport::TestCase
@storage_manager.store_file(StringIO.new("data"), @post, :preview)
@storage_manager.store_file(StringIO.new("data"), @post, :large)
@storage_manager.store_file(StringIO.new("data"), @post, :original)
subdir = "#{@post.md5[0..1]}/#{@post.md5[2..3]}"
@file_path = "#{@temp_dir}/preview/#{@post.md5}.jpg"
@large_file_path = "#{@temp_dir}/sample/sample-#{@post.md5}.jpg"
@preview_file_path = "#{@temp_dir}/#{@post.md5}.#{@post.file_ext}"
@file_path = "#{@temp_dir}/preview/#{subdir}/#{@post.md5}.jpg"
@large_file_path = "#{@temp_dir}/sample/#{subdir}/sample-#{@post.md5}.jpg"
@preview_file_path = "#{@temp_dir}/original/#{subdir}/#{@post.md5}.#{@post.file_ext}"
end
should "store the files at the correct path" do
@@ -108,10 +109,11 @@ class StorageManagerTest < ActiveSupport::TestCase
should "return the correct urls" do
@post = FactoryBot.create(:post, file_ext: "png")
@storage_manager.stubs(:tagged_filenames).returns(false)
subdir = "#{@post.md5[0..1]}/#{@post.md5[2..3]}"
assert_equal("/data/#{@post.md5}.png", @storage_manager.file_url(@post, :original))
assert_equal("/data/sample/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large))
assert_equal("/data/preview/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview))
assert_equal("/data/original/#{subdir}/#{@post.md5}.png", @storage_manager.file_url(@post, :original))
assert_equal("/data/sample/#{subdir}/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large))
assert_equal("/data/preview/#{subdir}/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview))
end
should "return the correct url for flash files" do
@@ -145,15 +147,15 @@ class StorageManagerTest < ActiveSupport::TestCase
@storage_manager.store_file(StringIO.new("post1"), @post1, :original)
@storage_manager.store_file(StringIO.new("post2"), @post2, :original)
assert(File.exist?("#{@temp_dir}/i1/#{@post1.md5}.png"))
assert(File.exist?("#{@temp_dir}/i2/#{@post2.md5}.png"))
assert(File.exist?("#{@temp_dir}/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png"))
assert(File.exist?("#{@temp_dir}/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png"))
end
end
context "#file_url method" do
should "generate /i1 urls for odd posts and /i2 urls for even posts" do
assert_equal("/i1/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
assert_equal("/i2/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
assert_equal("/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
assert_equal("/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
end
end
end