From 65e53b86b398f868f4d048e4f8d9a2a0cedf204f Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 29 Aug 2019 00:43:23 -0500 Subject: [PATCH] Drop support for /cache/tag.json. Drop support for https://danbooru.donmai.us/cache/tags.json. This was a nightly dump of the tags table that was originally added in #1012. It was never documented and never really used except for by the DanbooruUp extension. --- app/logical/api_cache_generator.rb | 33 ---------------- app/logical/danbooru_maintenance.rb | 1 - app/logical/remote_file_manager.rb | 54 -------------------------- config/danbooru_default_config.rb | 17 -------- lib/capistrano/tasks/symlink.rake | 6 --- test/unit/danbooru_maintenance_test.rb | 10 ----- 6 files changed, 121 deletions(-) delete mode 100644 app/logical/api_cache_generator.rb delete mode 100644 app/logical/remote_file_manager.rb diff --git a/app/logical/api_cache_generator.rb b/app/logical/api_cache_generator.rb deleted file mode 100644 index 404c72353..000000000 --- a/app/logical/api_cache_generator.rb +++ /dev/null @@ -1,33 +0,0 @@ -# donmai.us specific - -class ApiCacheGenerator - def generate_tag_cache - path = Danbooru.config.shared_dir_path - FileUtils.mkdir_p("#{path}/system/cache") - File.open("#{path}/system/cache/tags.json", "w") do |f| - f.print("[") - Tag.without_timeout do - Tag.find_each do |tag| - next unless tag.post_count > 0 - hash = { - "name" => tag.name, - "id" => tag.id, - "created_at" => tag.created_at, - "post_count" => tag.post_count, - "category" => tag.category - } - f.print(hash.to_json) - f.print(", ") - end - end - f.seek(-2, IO::SEEK_END) - f.print("]\n") - end - Zlib::GzipWriter.open("#{path}/system/cache/tags.json.gz") do |gz| - gz.write(IO.binread("#{path}/system/cache/tags.json")) - gz.close - end - RemoteFileManager.new("#{path}/system/cache/tags.json").distribute - RemoteFileManager.new("#{path}/system/cache/tags.json.gz").distribute - end -end diff --git a/app/logical/danbooru_maintenance.rb b/app/logical/danbooru_maintenance.rb index 1cb5cb388..8811dd6e1 100644 --- a/app/logical/danbooru_maintenance.rb +++ b/app/logical/danbooru_maintenance.rb @@ -13,7 +13,6 @@ module DanbooruMaintenance Delayed::Job.where('created_at < ?', 45.days.ago).delete_all PostVote.prune! CommentVote.prune! - ApiCacheGenerator.new.generate_tag_cache PostDisapproval.prune! ForumSubscription.process_all! TagAlias.update_cached_post_counts_for_all diff --git a/app/logical/remote_file_manager.rb b/app/logical/remote_file_manager.rb deleted file mode 100644 index 16669c9a3..000000000 --- a/app/logical/remote_file_manager.rb +++ /dev/null @@ -1,54 +0,0 @@ -require 'securerandom' - -class RemoteFileManager - attr_reader :path - - def initialize(path) - @path = path - end - - def distribute_to_archive(dest_url) - uri = URI.parse(dest_url) - dir_name = uri.host.split(".").first - uuid = SecureRandom.uuid - dest_path = "/var/www/#{dir_name}#{uri.path}" - temp_path = "/tmp/rfm-#{Danbooru.config.server_host}-#{uuid}" - - Net::SFTP.start(uri.host, Danbooru.config.archive_server_login) do |ftp| - ftp.upload!(path, temp_path) - begin - ftp.rename!(temp_path, dest_path) - rescue Net::SFTP::StatusException - ftp.remove!(dest_path) - ftp.rename!(temp_apth, dest_path) - end - end - end - - def distribute - uuid = SecureRandom.uuid - temp_path = "/tmp/rfm-#{Danbooru.config.server_host}-#{uuid}" - - Danbooru.config.other_server_hosts.each do |hostname| - Net::SFTP.start(hostname, Danbooru.config.remote_server_login) do |ftp| - ftp.upload!(path, temp_path) - begin - ftp.rename!(temp_path, path) - rescue Net::SFTP::StatusException - # this typically means the file already exists - # so delete and try renaming again - ftp.remove!(path) - ftp.rename!(temp_path, path) - end - end - end - end - - def delete - Danbooru.config.other_server_hosts.each do |hostname| - Net::SFTP.start(hostname, Danbooru.config.remote_server_login) do |ftp| - ftp.remove(path) - end - end - end -end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index ec69f7044..d3164f4f4 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -188,19 +188,6 @@ module Danbooru [server_host] end - # Names of other Danbooru servers. - def other_server_hosts - @other_server_hosts ||= all_server_hosts.reject {|x| x == server_host} - end - - def remote_server_login - "danbooru" - end - - def archive_server_login - "danbooru" - end - # The method to use for storing image files. def storage_manager # Store files on the local filesystem. @@ -474,10 +461,6 @@ module Danbooru %w[duplicate image_sample md5_mismatch resized upscaled downscaled] end - def shared_dir_path - "/var/www/danbooru2/shared" - end - def stripe_secret_key end diff --git a/lib/capistrano/tasks/symlink.rake b/lib/capistrano/tasks/symlink.rake index 656f41257..c646a6225 100644 --- a/lib/capistrano/tasks/symlink.rake +++ b/lib/capistrano/tasks/symlink.rake @@ -15,12 +15,6 @@ namespace :symlink do on roles(:app, :worker) do execute :rm, "-f", "#{release_path}/public/data" execute :ln, "-s", "#{deploy_to}/shared/data", "#{release_path}/public/data" - execute :mkdir, "-p", "#{release_path}/public/cache" - execute :mkdir, "-p", "#{deploy_to}/shared/system/cache" - execute :touch, "#{deploy_to}/shared/system/cache/tags.json" - execute :ln, "-s", "#{deploy_to}/shared/system/cache/tags.json", "#{release_path}/public/cache/tags.json" - execute :touch, "#{deploy_to}/shared/system/cache/tags.json.gz" - execute :ln, "-s", "#{deploy_to}/shared/system/cache/tags.json.gz", "#{release_path}/public/cache/tags.json.gz" end end end diff --git a/test/unit/danbooru_maintenance_test.rb b/test/unit/danbooru_maintenance_test.rb index c0190e7dd..3ff2133b4 100644 --- a/test/unit/danbooru_maintenance_test.rb +++ b/test/unit/danbooru_maintenance_test.rb @@ -3,19 +3,9 @@ require 'test_helper' class DanbooruMaintenanceTest < ActiveSupport::TestCase context "daily maintenance" do setup do - # have ApiCacheGenerator save files to a temp dir. - @temp_shared_dir_path = "/tmp/#{SecureRandom.uuid}" - Danbooru.config.stubs(:shared_dir_path).returns(@temp_shared_dir_path) - - FactoryBot.create(:tag, post_count: 1) # for ApiCacheGenerator FactoryBot.create(:admin_user) # for SuperVoter.init! end - teardown do - FileUtils.rm_rf(@temp_shared_dir_path) - Danbooru.config.unstub(:shared_dir_path) - end - should "work" do assert_nothing_raised { DanbooruMaintenance.daily } end