storage manager: remove hierarchical option.

Remove the `hierarchical` file storage option. This means that image
files are always stored in MD5-based subdirectories, like this:

   https://danbooru.donmai.us/data/original/f3/a7/f3a70a89c350b5ed4db22dbb25b934bb.jpg
   https://danbooru.donmai.us/data/sample/f3/a7/sample-f3a70a89c350b5ed4db22dbb25b934bb.jpg
   https://danbooru.donmai.us/data/preview/f3/a7/f3a70a89c350b5ed4db22dbb25b934bb.jpg

instead of in a single flat directory, like this:

   https://danbooru.donmai.us/data/original/f3a70a89c350b5ed4db22dbb25b934bb.jpg

This option is removed because storing files in a single directory is a
bad idea for large installations, and migrating from a single directory
to subdirectories later is a pain.

Downstream boorus who still have files in the old layout can migrate by
running this script:

   `./script/fixes/077_symlink_subdirectories.rb`

This will create symlinks that redirect the 00-ff subdirectories back to
the current directory, so that you can still store files in a single
directory, but use URLs containing subdirectories.

You should also make sure to remove the `hierarchical` option from
`storage_manager` in `config/danbooru_local_config.rb` if you set it
there.

test/test_helper.rb

@@ -49,9 +49,9 @@ class ActiveSupport::TestCase
     Socket.stubs(:gethostname).returns("www.example.com")
 
     @temp_dir = Dir.mktmpdir("danbooru-temp-")
-    storage_manager = StorageManager::Local.new(base_dir: @temp_dir)
+    storage_manager = StorageManager::Local.new(base_url: "https://www.example.com/data", base_dir: @temp_dir)
     Danbooru.config.stubs(:storage_manager).returns(storage_manager)
-    Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
+    Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new(base_url: "/", base_dir: "/"))
   end
 
   teardown do

test/unit/post_test.rb

@@ -1972,14 +1972,14 @@ class PostTest < ActiveSupport::TestCase
 
   context "URLs:" do
     should "generate the correct urls for animated gifs" do
-      manager = StorageManager::Local.new(base_url: "https://test.com/data")
+      manager = StorageManager::Local.new(base_url: "https://test.com/data", base_dir: "/")
       Danbooru.config.stubs(:storage_manager).returns(manager)
 
       @post = build(:post, md5: "deadbeef", file_ext: "gif", tag_string: "animated_gif")
 
-      assert_equal("https://test.com/data/preview/deadbeef.jpg", @post.preview_file_url)
-      assert_equal("https://test.com/data/deadbeef.gif", @post.large_file_url)
-      assert_equal("https://test.com/data/deadbeef.gif", @post.file_url)
+      assert_equal("https://test.com/data/preview/de/ad/deadbeef.jpg", @post.preview_file_url)
+      assert_equal("https://test.com/data/original/de/ad/deadbeef.gif", @post.large_file_url)
+      assert_equal("https://test.com/data/original/de/ad/deadbeef.gif", @post.file_url)
     end
   end
 

test/unit/storage_manager_test.rb

@@ -81,10 +81,11 @@ class StorageManagerTest < ActiveSupport::TestCase
         @storage_manager.store_file(StringIO.new("data"), @post, :preview)
         @storage_manager.store_file(StringIO.new("data"), @post, :large)
         @storage_manager.store_file(StringIO.new("data"), @post, :original)
+        subdir = "#{@post.md5[0..1]}/#{@post.md5[2..3]}"
 
-        @file_path = "#{@temp_dir}/preview/#{@post.md5}.jpg"
-        @large_file_path = "#{@temp_dir}/sample/sample-#{@post.md5}.jpg"
-        @preview_file_path = "#{@temp_dir}/#{@post.md5}.#{@post.file_ext}"
+        @file_path = "#{@temp_dir}/preview/#{subdir}/#{@post.md5}.jpg"
+        @large_file_path = "#{@temp_dir}/sample/#{subdir}/sample-#{@post.md5}.jpg"
+        @preview_file_path = "#{@temp_dir}/original/#{subdir}/#{@post.md5}.#{@post.file_ext}"
       end
 
       should "store the files at the correct path" do
@@ -108,10 +109,11 @@ class StorageManagerTest < ActiveSupport::TestCase
       should "return the correct urls" do
         @post = FactoryBot.create(:post, file_ext: "png")
         @storage_manager.stubs(:tagged_filenames).returns(false)
+        subdir = "#{@post.md5[0..1]}/#{@post.md5[2..3]}"
 
-        assert_equal("/data/#{@post.md5}.png", @storage_manager.file_url(@post, :original))
-        assert_equal("/data/sample/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large))
-        assert_equal("/data/preview/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview))
+        assert_equal("/data/original/#{subdir}/#{@post.md5}.png", @storage_manager.file_url(@post, :original))
+        assert_equal("/data/sample/#{subdir}/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large))
+        assert_equal("/data/preview/#{subdir}/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview))
       end
 
       should "return the correct url for flash files" do
@@ -145,15 +147,15 @@ class StorageManagerTest < ActiveSupport::TestCase
         @storage_manager.store_file(StringIO.new("post1"), @post1, :original)
         @storage_manager.store_file(StringIO.new("post2"), @post2, :original)
 
-        assert(File.exist?("#{@temp_dir}/i1/#{@post1.md5}.png"))
-        assert(File.exist?("#{@temp_dir}/i2/#{@post2.md5}.png"))
+        assert(File.exist?("#{@temp_dir}/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png"))
+        assert(File.exist?("#{@temp_dir}/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png"))
       end
     end
 
     context "#file_url method" do
       should "generate /i1 urls for odd posts and /i2 urls for even posts" do
-        assert_equal("/i1/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
-        assert_equal("/i2/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
+        assert_equal("/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
+        assert_equal("/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
       end
     end
   end