maintenance: break maintenance tasks into individual jobs.

Break the hourly/daily/weekly/monthly maintenance tasks down into
individual delayed jobs. This way if one task fails, it won't prevent
other tasks from running. Also, jobs can be run in parallel, and can be
individually retried if they fail.

app/jobs/bigquery_export_all_job.rb

@@ -0,0 +1,6 @@
+# A job that runs daily to export all tables to BigQuery. Spawned by {DanbooruMaintenance}.
+class BigqueryExportAllJob < ApplicationJob
+  def perform
+    BigqueryExportService.async_export_all!
+  end
+end

app/jobs/dmail_inactive_approvers_job.rb

@@ -0,0 +1,7 @@
+# A job that runs weekly to warn inactive approvers before they're demoted.
+# Spawned by {DanbooruMaintenance}.
+class DmailInactiveApproversJob < ApplicationJob
+  def perform
+    ApproverPruner.dmail_inactive_approvers!
+  end
+end

app/jobs/prune_approvers_job.rb

@@ -0,0 +1,7 @@
+# A job that runs monthly to demote all inactive approvers. Spawned by
+# {DanbooruMaintenance}.
+class PruneApproversJob < ApplicationJob
+  def perform
+    ApproverPruner.prune!
+  end
+end

app/jobs/prune_bans_job.rb

@@ -0,0 +1,7 @@
+# A job that runs daily to remove expired bans. Spawned by
+# {DanbooruMaintenance}.
+class PruneBansJob < ApplicationJob
+  def perform
+    Ban.prune!
+  end
+end

app/jobs/prune_bulk_update_requests_job.rb

@@ -0,0 +1,8 @@
+# A job that runs daily to reject expired bulk update requests. Spawned by
+# {DanbooruMaintenance}.
+class PruneBulkUpdateRequestsJob < ApplicationJob
+  def perform
+    BulkUpdateRequestPruner.warn_old
+    BulkUpdateRequestPruner.reject_expired
+  end
+end

app/jobs/prune_delayed_jobs_job.rb

@@ -0,0 +1,7 @@
+# A job that runs daily to delete all stale delayed jobs. Spawned by
+# {DanbooruMaintenance}.
+class PruneDelayedJobsJob < ApplicationJob
+  def perform
+    Delayed::Job.where("created_at < ?", 45.days.ago).delete_all
+  end
+end

app/jobs/prune_post_disapprovals_job.rb

@@ -0,0 +1,7 @@
+# A job that runs daily to remove old post disapprovals. Spawned by
+# {DanbooruMaintenance}.
+class PrunePostDisapprovalsJob < ApplicationJob
+  def perform
+    PostDisapproval.prune!
+  end
+end

app/jobs/prune_posts_job.rb

@@ -0,0 +1,7 @@
+# A job that runs hourly to delete all expired pending, flagged, and appealed
+# posts. Spawned by {DanbooruMaintenance}.
+class PrunePostsJob < ApplicationJob
+  def perform
+    PostPruner.prune!
+  end
+end

app/jobs/prune_rate_limits_job.rb

@@ -0,0 +1,7 @@
+# A job that runs hourly to delete all state rate limit objects from the
+# database. Spawned by {DanbooruMaintenance}.
+class PruneRateLimitsJob < ApplicationJob
+  def perform
+    RateLimit.prune!
+  end
+end

app/jobs/prune_uploads_job.rb

@@ -0,0 +1,7 @@
+# A job that runs hourly to delete all completed, stale, or failed uploads.
+# Spawned by {DanbooruMaintenance}.
+class PruneUploadsJob < ApplicationJob
+  def perform
+    Upload.prune!
+  end
+end

app/jobs/regenerate_post_counts_job.rb

@@ -0,0 +1,10 @@
+# A job that runs hourly to fix all incorrect tag counts.
+# Spawned by {DanbooruMaintenance}.
+class RegeneratePostCountsJob < ApplicationJob
+  def perform
+    updated_tags = Tag.regenerate_post_counts!
+    updated_tags.each do |tag|
+      DanbooruLogger.info("Updated tag count", tag.attributes)
+    end
+  end
+end

app/jobs/retire_tag_relationships_job.rb

@@ -0,0 +1,7 @@
+# A job that runs weekly to retire inactive aliases and implications. Spawned
+# by {DanbooruMaintenance}.
+class RetireTagRelationshipsJob < ApplicationJob
+  def perform
+    TagRelationshipRetirementService.find_and_retire!
+  end
+end

app/jobs/vacuum_database_job.rb

@@ -0,0 +1,8 @@
+# A job that runs daily to vacuum the database. Spawned by {DanbooruMaintenance}.
+class VacuumDatabaseJob < ApplicationJob
+  def perform
+    # We can't perform vacuum inside a transaction. This happens during tests.
+    return if ApplicationRecord.connection.transaction_open?
+    ApplicationRecord.connection.execute("vacuum analyze")
+  end
+end

app/logical/danbooru_maintenance.rb

@@ -2,45 +2,34 @@ module DanbooruMaintenance
   module_function
 
   def hourly
-    safely { Upload.prune! }
-    safely { PostPruner.prune! }
-    safely { RateLimit.prune! }
-    safely { regenerate_post_counts! }
+    queue PruneUploadsJob
+    queue PrunePostsJob
+    queue PruneRateLimitsJob
+    queue RegeneratePostCountsJob
   end
 
   def daily
-    safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all }
-    safely { PostDisapproval.prune! }
-    safely { BulkUpdateRequestPruner.warn_old }
-    safely { BulkUpdateRequestPruner.reject_expired }
-    safely { Ban.prune! }
-    safely { BigqueryExportService.async_export_all! }
-    safely { ActiveRecord::Base.connection.execute("vacuum analyze") unless Rails.env.test? }
+    queue PruneDelayedJobsJob
+    queue PrunePostDisapprovalsJob
+    queue PruneBulkUpdateRequestsJob
+    queue PruneBansJob
+    queue BigqueryExportAllJob
+    queue VacuumDatabaseJob
   end
 
   def weekly
-    safely { TagRelationshipRetirementService.find_and_retire! }
-    safely { ApproverPruner.dmail_inactive_approvers! }
+    queue RetireTagRelationshipsJob
+    queue DmailInactiveApproversJob
   end
 
   def monthly
-    safely { ApproverPruner.prune! }
+    queue PruneApproversJob
   end
 
-  def regenerate_post_counts!
-    updated_tags = Tag.regenerate_post_counts!
-    updated_tags.each do |tag|
-      DanbooruLogger.info("Updated tag count", tag.attributes)
-    end
-  end
-
-  def safely(&block)
-    ActiveRecord::Base.connection.execute("set statement_timeout = 0")
-
-    CurrentUser.scoped(User.system, "127.0.0.1") do
-      yield
-    end
-  rescue StandardError => exception
+  def queue(job)
+    DanbooruLogger.info("Queueing #{job.name}")
+    job.perform_later
+  rescue Exception # rubocop:disable Lint/RescueException
     DanbooruLogger.log(exception)
     raise exception if Rails.env.test?
   end

test/jobs/prune_bans_job_test.rb

@@ -0,0 +1,18 @@
+require 'test_helper'
+
+class PruneBansJobTest < ActiveJob::TestCase
+  context "PruneBansJob" do
+    should "prune all expired bans" do
+      @expired_ban = travel_to(1.month.ago) { create(:ban, duration: 1.week) }
+      @unexpired_ban = create(:ban, duration: 1.week)
+
+      assert_equal(true, @expired_ban.user.is_banned?)
+      assert_equal(true, @unexpired_ban.user.is_banned?)
+
+      PruneBansJob.perform_now
+
+      assert_equal(false, @expired_ban.user.reload.is_banned?)
+      assert_equal(true, @unexpired_ban.user.reload.is_banned?)
+    end
+  end
+end

test/jobs/prune_posts_job_test.rb

@@ -0,0 +1,22 @@
+require 'test_helper'
+
+class PrunePostsJobTest < ActiveJob::TestCase
+  context "PrunePostsJob" do
+    should "prune expired posts" do
+      @pending = create(:post, is_pending: true, created_at: 5.days.ago)
+      @flagged = create(:post, is_flagged: true, created_at: 5.days.ago)
+      @appealed = create(:post, is_deleted: true, created_at: 5.days.ago)
+
+      @flag = create(:post_flag, post: @flagged, created_at: 4.days.ago)
+      @appeal = create(:post_appeal, post: @appealed, created_at: 4.days.ago)
+
+      PrunePostsJob.perform_now
+
+      assert_equal(true, @pending.reload.is_deleted?)
+      assert_equal(true, @flagged.reload.is_deleted?)
+      assert_equal(true, @appealed.reload.is_deleted?)
+      assert_equal(true, @flag.reload.succeeded?)
+      assert_equal(true, @appeal.reload.rejected?)
+    end
+  end
+end

test/jobs/prune_rate_limits_job_test.rb

@@ -0,0 +1,13 @@
+require 'test_helper'
+
+class PruneRateLimitsJobTest < ActiveJob::TestCase
+  context "PruneRateLimitsJob" do
+    should "prune all stale rate limits" do
+      travel_to(2.hours.ago) { create(:rate_limit) }
+
+      assert_equal(1, RateLimit.count)
+      PruneRateLimitsJob.perform_now
+      assert_equal(0, RateLimit.count)
+    end
+  end
+end

test/jobs/prune_uploads_job_test.rb

@@ -0,0 +1,19 @@
+require 'test_helper'
+
+class PruneUploadsJobTest < ActiveJob::TestCase
+  context "PruneUploadsJob" do
+    should "prune all old uploads" do
+      @uploader = create(:user)
+
+      as(@uploader) do
+        @completed_upload = travel_to(2.hours.ago) { create(:upload, uploader: @uploader, status: "completed") }
+        @stale_upload = travel_to(2.days.ago) { create(:upload, uploader: @uploader, status: "preprocessed") }
+        @failed_upload = travel_to(4.days.ago) { create(:upload, uploader: @uploader, status: "error") }
+      end
+
+      assert_equal(3, Upload.count)
+      PruneUploadsJob.perform_now
+      assert_equal(0, Upload.count)
+    end
+  end
+end

test/jobs/regenerate_post_counts_job_test.rb

@@ -0,0 +1,18 @@
+require 'test_helper'
+
+class RegeneratePostCountsJobTest < ActiveJob::TestCase
+  context "RegeneratePostCountsJob" do
+    should "regenerate all incorrect tag post counts" do
+      tag1 = create(:tag, name: "touhou", post_count: -10)
+      tag2 = create(:tag, name: "bkub", post_count: 10)
+      tag3 = create(:tag, name: "chen", post_count: 10)
+      post = create(:post, tag_string: "touhou bkub")
+
+      RegeneratePostCountsJob.perform_now
+
+      assert_equal(1, Tag.find_by_name!("touhou").post_count)
+      assert_equal(1, Tag.find_by_name!("bkub").post_count)
+      assert_equal(0, Tag.find_by_name!("chen").post_count)
+    end
+  end
+end

test/unit/danbooru_maintenance_test.rb

@@ -3,38 +3,36 @@ require 'test_helper'
 class DanbooruMaintenanceTest < ActiveSupport::TestCase
   context "hourly maintenance" do
     should "work" do
-      assert_nothing_raised { DanbooruMaintenance.hourly }
-    end
-
-    should "prune expired posts" do
-      @pending = create(:post, is_pending: true, created_at: 5.days.ago)
-      @flagged = create(:post, is_flagged: true, created_at: 5.days.ago)
-      @appealed = create(:post, is_deleted: true, created_at: 5.days.ago)
-
-      @flag = create(:post_flag, post: @flagged, created_at: 4.days.ago)
-      @appeal = create(:post_appeal, post: @appealed, created_at: 4.days.ago)
-
-      DanbooruMaintenance.hourly
-
-      assert_equal(true, @pending.reload.is_deleted?)
-      assert_equal(true, @flagged.reload.is_deleted?)
-      assert_equal(true, @appealed.reload.is_deleted?)
-      assert_equal(true, @flag.reload.succeeded?)
-      assert_equal(true, @appeal.reload.rejected?)
+      assert_nothing_raised do
+        DanbooruMaintenance.hourly
+        perform_enqueued_jobs
+      end
     end
   end
 
-  context "hourly maintenance" do
-    context "when pruning bans" do
-      should "clear the is_banned flag for users who are no longer banned" do
-        banner = FactoryBot.create(:admin_user)
-        user = FactoryBot.create(:user)
+  context "daily maintenance" do
+    should "work" do
+      assert_nothing_raised do
+        DanbooruMaintenance.daily
+        perform_enqueued_jobs
+      end
+    end
+  end
 
-        as(banner) { create(:ban, user: user, banner: banner, duration: 1) }
+  context "weekly maintenance" do
+    should "work" do
+      assert_nothing_raised do
+        DanbooruMaintenance.weekly
+        perform_enqueued_jobs
+      end
+    end
+  end
 
-        assert_equal(true, user.reload.is_banned)
-        travel_to(2.days.from_now) { DanbooruMaintenance.daily }
-        assert_equal(false, user.reload.is_banned)
+  context "monthly maintenance" do
+    should "work" do
+      assert_nothing_raised do
+        DanbooruMaintenance.monthly
+        perform_enqueued_jobs
       end
     end
   end

test/unit/tag_test.rb

@@ -194,19 +194,4 @@ class TagTest < ActiveSupport::TestCase
       end
     end
   end
-
-  context "A tag with an incorrect post count" do
-    should "be fixed" do
-      tag1 = FactoryBot.create(:tag, name: "touhou", post_count: -10)
-      tag2 = FactoryBot.create(:tag, name: "bkub", post_count: 10)
-      tag3 = FactoryBot.create(:tag, name: "chen", post_count: 10)
-      post = FactoryBot.create(:post, tag_string: "touhou bkub")
-
-      tags = Tag.regenerate_post_counts!
-      assert_equal(3, tags.size)
-      assert_equal(1, tag1.reload.post_count)
-      assert_equal(1, tag2.reload.post_count)
-      assert_equal(0, tag3.reload.post_count)
-    end
-  end
 end