From 52bf4a3a6b41cf6156540abe79508bda09bbd69a Mon Sep 17 00:00:00 2001
From: evazion <noizave@gmail.com>
Date: Sun, 26 Sep 2021 20:05:39 -0500
Subject: [PATCH] maintenance: break maintenance tasks into individual jobs.

Break the hourly/daily/weekly/monthly maintenance tasks down into
individual delayed jobs. This way if one task fails, it won't prevent
other tasks from running. Also, jobs can be run in parallel, and can be
individually retried if they fail.
---
 app/jobs/bigquery_export_all_job.rb          |  6 +++
 app/jobs/dmail_inactive_approvers_job.rb     |  7 +++
 app/jobs/prune_approvers_job.rb              |  7 +++
 app/jobs/prune_bans_job.rb                   |  7 +++
 app/jobs/prune_bulk_update_requests_job.rb   |  8 +++
 app/jobs/prune_delayed_jobs_job.rb           |  7 +++
 app/jobs/prune_post_disapprovals_job.rb      |  7 +++
 app/jobs/prune_posts_job.rb                  |  7 +++
 app/jobs/prune_rate_limits_job.rb            |  7 +++
 app/jobs/prune_uploads_job.rb                |  7 +++
 app/jobs/regenerate_post_counts_job.rb       | 10 ++++
 app/jobs/retire_tag_relationships_job.rb     |  7 +++
 app/jobs/vacuum_database_job.rb              |  8 +++
 app/logical/danbooru_maintenance.rb          | 45 +++++++----------
 test/jobs/prune_bans_job_test.rb             | 18 +++++++
 test/jobs/prune_posts_job_test.rb            | 22 +++++++++
 test/jobs/prune_rate_limits_job_test.rb      | 13 +++++
 test/jobs/prune_uploads_job_test.rb          | 19 +++++++
 test/jobs/regenerate_post_counts_job_test.rb | 18 +++++++
 test/unit/danbooru_maintenance_test.rb       | 52 ++++++++++----------
 test/unit/tag_test.rb                        | 15 ------
 21 files changed, 227 insertions(+), 70 deletions(-)
 create mode 100644 app/jobs/bigquery_export_all_job.rb
 create mode 100644 app/jobs/dmail_inactive_approvers_job.rb
 create mode 100644 app/jobs/prune_approvers_job.rb
 create mode 100644 app/jobs/prune_bans_job.rb
 create mode 100644 app/jobs/prune_bulk_update_requests_job.rb
 create mode 100644 app/jobs/prune_delayed_jobs_job.rb
 create mode 100644 app/jobs/prune_post_disapprovals_job.rb
 create mode 100644 app/jobs/prune_posts_job.rb
 create mode 100644 app/jobs/prune_rate_limits_job.rb
 create mode 100644 app/jobs/prune_uploads_job.rb
 create mode 100644 app/jobs/regenerate_post_counts_job.rb
 create mode 100644 app/jobs/retire_tag_relationships_job.rb
 create mode 100644 app/jobs/vacuum_database_job.rb
 create mode 100644 test/jobs/prune_bans_job_test.rb
 create mode 100644 test/jobs/prune_posts_job_test.rb
 create mode 100644 test/jobs/prune_rate_limits_job_test.rb
 create mode 100644 test/jobs/prune_uploads_job_test.rb
 create mode 100644 test/jobs/regenerate_post_counts_job_test.rb

diff --git a/app/jobs/bigquery_export_all_job.rb b/app/jobs/bigquery_export_all_job.rb
new file mode 100644
index 000000000..195734ff0
--- /dev/null
+++ b/app/jobs/bigquery_export_all_job.rb
@@ -0,0 +1,6 @@
+# A job that runs daily to export all tables to BigQuery. Spawned by {DanbooruMaintenance}.
+class BigqueryExportAllJob < ApplicationJob
+  def perform
+    BigqueryExportService.async_export_all!
+  end
+end
diff --git a/app/jobs/dmail_inactive_approvers_job.rb b/app/jobs/dmail_inactive_approvers_job.rb
new file mode 100644
index 000000000..588ef2ecd
--- /dev/null
+++ b/app/jobs/dmail_inactive_approvers_job.rb
@@ -0,0 +1,7 @@
+# A job that runs weekly to warn inactive approvers before they're demoted.
+# Spawned by {DanbooruMaintenance}.
+class DmailInactiveApproversJob < ApplicationJob
+  def perform
+    ApproverPruner.dmail_inactive_approvers!
+  end
+end
diff --git a/app/jobs/prune_approvers_job.rb b/app/jobs/prune_approvers_job.rb
new file mode 100644
index 000000000..5dcc80411
--- /dev/null
+++ b/app/jobs/prune_approvers_job.rb
@@ -0,0 +1,7 @@
+# A job that runs monthly to demote all inactive approvers. Spawned by
+# {DanbooruMaintenance}.
+class PruneApproversJob < ApplicationJob
+  def perform
+    ApproverPruner.prune!
+  end
+end
diff --git a/app/jobs/prune_bans_job.rb b/app/jobs/prune_bans_job.rb
new file mode 100644
index 000000000..952ea96bd
--- /dev/null
+++ b/app/jobs/prune_bans_job.rb
@@ -0,0 +1,7 @@
+# A job that runs daily to remove expired bans. Spawned by
+# {DanbooruMaintenance}.
+class PruneBansJob < ApplicationJob
+  def perform
+    Ban.prune!
+  end
+end
diff --git a/app/jobs/prune_bulk_update_requests_job.rb b/app/jobs/prune_bulk_update_requests_job.rb
new file mode 100644
index 000000000..5d6fdfb39
--- /dev/null
+++ b/app/jobs/prune_bulk_update_requests_job.rb
@@ -0,0 +1,8 @@
+# A job that runs daily to reject expired bulk update requests. Spawned by
+# {DanbooruMaintenance}.
+class PruneBulkUpdateRequestsJob < ApplicationJob
+  def perform
+    BulkUpdateRequestPruner.warn_old
+    BulkUpdateRequestPruner.reject_expired
+  end
+end
diff --git a/app/jobs/prune_delayed_jobs_job.rb b/app/jobs/prune_delayed_jobs_job.rb
new file mode 100644
index 000000000..caa986b6f
--- /dev/null
+++ b/app/jobs/prune_delayed_jobs_job.rb
@@ -0,0 +1,7 @@
+# A job that runs daily to delete all stale delayed jobs. Spawned by
+# {DanbooruMaintenance}.
+class PruneDelayedJobsJob < ApplicationJob
+  def perform
+    Delayed::Job.where("created_at < ?", 45.days.ago).delete_all
+  end
+end
diff --git a/app/jobs/prune_post_disapprovals_job.rb b/app/jobs/prune_post_disapprovals_job.rb
new file mode 100644
index 000000000..f26f12e6a
--- /dev/null
+++ b/app/jobs/prune_post_disapprovals_job.rb
@@ -0,0 +1,7 @@
+# A job that runs daily to remove old post disapprovals. Spawned by
+# {DanbooruMaintenance}.
+class PrunePostDisapprovalsJob < ApplicationJob
+  def perform
+    PostDisapproval.prune!
+  end
+end
diff --git a/app/jobs/prune_posts_job.rb b/app/jobs/prune_posts_job.rb
new file mode 100644
index 000000000..2fcab7739
--- /dev/null
+++ b/app/jobs/prune_posts_job.rb
@@ -0,0 +1,7 @@
+# A job that runs hourly to delete all expired pending, flagged, and appealed
+# posts. Spawned by {DanbooruMaintenance}.
+class PrunePostsJob < ApplicationJob
+  def perform
+    PostPruner.prune!
+  end
+end
diff --git a/app/jobs/prune_rate_limits_job.rb b/app/jobs/prune_rate_limits_job.rb
new file mode 100644
index 000000000..d9291d981
--- /dev/null
+++ b/app/jobs/prune_rate_limits_job.rb
@@ -0,0 +1,7 @@
+# A job that runs hourly to delete all state rate limit objects from the
+# database. Spawned by {DanbooruMaintenance}.
+class PruneRateLimitsJob < ApplicationJob
+  def perform
+    RateLimit.prune!
+  end
+end
diff --git a/app/jobs/prune_uploads_job.rb b/app/jobs/prune_uploads_job.rb
new file mode 100644
index 000000000..378abf2a6
--- /dev/null
+++ b/app/jobs/prune_uploads_job.rb
@@ -0,0 +1,7 @@
+# A job that runs hourly to delete all completed, stale, or failed uploads.
+# Spawned by {DanbooruMaintenance}.
+class PruneUploadsJob < ApplicationJob
+  def perform
+    Upload.prune!
+  end
+end
diff --git a/app/jobs/regenerate_post_counts_job.rb b/app/jobs/regenerate_post_counts_job.rb
new file mode 100644
index 000000000..0cd7e9bc0
--- /dev/null
+++ b/app/jobs/regenerate_post_counts_job.rb
@@ -0,0 +1,10 @@
+# A job that runs hourly to fix all incorrect tag counts.
+# Spawned by {DanbooruMaintenance}.
+class RegeneratePostCountsJob < ApplicationJob
+  def perform
+    updated_tags = Tag.regenerate_post_counts!
+    updated_tags.each do |tag|
+      DanbooruLogger.info("Updated tag count", tag.attributes)
+    end
+  end
+end
diff --git a/app/jobs/retire_tag_relationships_job.rb b/app/jobs/retire_tag_relationships_job.rb
new file mode 100644
index 000000000..7b2e64f7f
--- /dev/null
+++ b/app/jobs/retire_tag_relationships_job.rb
@@ -0,0 +1,7 @@
+# A job that runs weekly to retire inactive aliases and implications. Spawned
+# by {DanbooruMaintenance}.
+class RetireTagRelationshipsJob < ApplicationJob
+  def perform
+    TagRelationshipRetirementService.find_and_retire!
+  end
+end
diff --git a/app/jobs/vacuum_database_job.rb b/app/jobs/vacuum_database_job.rb
new file mode 100644
index 000000000..34eb521b1
--- /dev/null
+++ b/app/jobs/vacuum_database_job.rb
@@ -0,0 +1,8 @@
+# A job that runs daily to vacuum the database. Spawned by {DanbooruMaintenance}.
+class VacuumDatabaseJob < ApplicationJob
+  def perform
+    # We can't perform vacuum inside a transaction. This happens during tests.
+    return if ApplicationRecord.connection.transaction_open?
+    ApplicationRecord.connection.execute("vacuum analyze")
+  end
+end
diff --git a/app/logical/danbooru_maintenance.rb b/app/logical/danbooru_maintenance.rb
index 19c5cd841..58f36187b 100644
--- a/app/logical/danbooru_maintenance.rb
+++ b/app/logical/danbooru_maintenance.rb
@@ -2,45 +2,34 @@ module DanbooruMaintenance
   module_function
 
   def hourly
-    safely { Upload.prune! }
-    safely { PostPruner.prune! }
-    safely { RateLimit.prune! }
-    safely { regenerate_post_counts! }
+    queue PruneUploadsJob
+    queue PrunePostsJob
+    queue PruneRateLimitsJob
+    queue RegeneratePostCountsJob
   end
 
   def daily
-    safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all }
-    safely { PostDisapproval.prune! }
-    safely { BulkUpdateRequestPruner.warn_old }
-    safely { BulkUpdateRequestPruner.reject_expired }
-    safely { Ban.prune! }
-    safely { BigqueryExportService.async_export_all! }
-    safely { ActiveRecord::Base.connection.execute("vacuum analyze") unless Rails.env.test? }
+    queue PruneDelayedJobsJob
+    queue PrunePostDisapprovalsJob
+    queue PruneBulkUpdateRequestsJob
+    queue PruneBansJob
+    queue BigqueryExportAllJob
+    queue VacuumDatabaseJob
   end
 
   def weekly
-    safely { TagRelationshipRetirementService.find_and_retire! }
-    safely { ApproverPruner.dmail_inactive_approvers! }
+    queue RetireTagRelationshipsJob
+    queue DmailInactiveApproversJob
   end
 
   def monthly
-    safely { ApproverPruner.prune! }
+    queue PruneApproversJob
   end
 
-  def regenerate_post_counts!
-    updated_tags = Tag.regenerate_post_counts!
-    updated_tags.each do |tag|
-      DanbooruLogger.info("Updated tag count", tag.attributes)
-    end
-  end
-
-  def safely(&block)
-    ActiveRecord::Base.connection.execute("set statement_timeout = 0")
-
-    CurrentUser.scoped(User.system, "127.0.0.1") do
-      yield
-    end
-  rescue StandardError => exception
+  def queue(job)
+    DanbooruLogger.info("Queueing #{job.name}")
+    job.perform_later
+  rescue Exception # rubocop:disable Lint/RescueException
     DanbooruLogger.log(exception)
     raise exception if Rails.env.test?
   end
diff --git a/test/jobs/prune_bans_job_test.rb b/test/jobs/prune_bans_job_test.rb
new file mode 100644
index 000000000..7a456b55a
--- /dev/null
+++ b/test/jobs/prune_bans_job_test.rb
@@ -0,0 +1,18 @@
+require 'test_helper'
+
+class PruneBansJobTest < ActiveJob::TestCase
+  context "PruneBansJob" do
+    should "prune all expired bans" do
+      @expired_ban = travel_to(1.month.ago) { create(:ban, duration: 1.week) }
+      @unexpired_ban = create(:ban, duration: 1.week)
+
+      assert_equal(true, @expired_ban.user.is_banned?)
+      assert_equal(true, @unexpired_ban.user.is_banned?)
+
+      PruneBansJob.perform_now
+
+      assert_equal(false, @expired_ban.user.reload.is_banned?)
+      assert_equal(true, @unexpired_ban.user.reload.is_banned?)
+    end
+  end
+end
diff --git a/test/jobs/prune_posts_job_test.rb b/test/jobs/prune_posts_job_test.rb
new file mode 100644
index 000000000..ce568c924
--- /dev/null
+++ b/test/jobs/prune_posts_job_test.rb
@@ -0,0 +1,22 @@
+require 'test_helper'
+
+class PrunePostsJobTest < ActiveJob::TestCase
+  context "PrunePostsJob" do
+    should "prune expired posts" do
+      @pending = create(:post, is_pending: true, created_at: 5.days.ago)
+      @flagged = create(:post, is_flagged: true, created_at: 5.days.ago)
+      @appealed = create(:post, is_deleted: true, created_at: 5.days.ago)
+
+      @flag = create(:post_flag, post: @flagged, created_at: 4.days.ago)
+      @appeal = create(:post_appeal, post: @appealed, created_at: 4.days.ago)
+
+      PrunePostsJob.perform_now
+
+      assert_equal(true, @pending.reload.is_deleted?)
+      assert_equal(true, @flagged.reload.is_deleted?)
+      assert_equal(true, @appealed.reload.is_deleted?)
+      assert_equal(true, @flag.reload.succeeded?)
+      assert_equal(true, @appeal.reload.rejected?)
+    end
+  end
+end
diff --git a/test/jobs/prune_rate_limits_job_test.rb b/test/jobs/prune_rate_limits_job_test.rb
new file mode 100644
index 000000000..8868f242d
--- /dev/null
+++ b/test/jobs/prune_rate_limits_job_test.rb
@@ -0,0 +1,13 @@
+require 'test_helper'
+
+class PruneRateLimitsJobTest < ActiveJob::TestCase
+  context "PruneRateLimitsJob" do
+    should "prune all stale rate limits" do
+      travel_to(2.hours.ago) { create(:rate_limit) }
+
+      assert_equal(1, RateLimit.count)
+      PruneRateLimitsJob.perform_now
+      assert_equal(0, RateLimit.count)
+    end
+  end
+end
diff --git a/test/jobs/prune_uploads_job_test.rb b/test/jobs/prune_uploads_job_test.rb
new file mode 100644
index 000000000..4446229f4
--- /dev/null
+++ b/test/jobs/prune_uploads_job_test.rb
@@ -0,0 +1,19 @@
+require 'test_helper'
+
+class PruneUploadsJobTest < ActiveJob::TestCase
+  context "PruneUploadsJob" do
+    should "prune all old uploads" do
+      @uploader = create(:user)
+
+      as(@uploader) do
+        @completed_upload = travel_to(2.hours.ago) { create(:upload, uploader: @uploader, status: "completed") }
+        @stale_upload = travel_to(2.days.ago) { create(:upload, uploader: @uploader, status: "preprocessed") }
+        @failed_upload = travel_to(4.days.ago) { create(:upload, uploader: @uploader, status: "error") }
+      end
+
+      assert_equal(3, Upload.count)
+      PruneUploadsJob.perform_now
+      assert_equal(0, Upload.count)
+    end
+  end
+end
diff --git a/test/jobs/regenerate_post_counts_job_test.rb b/test/jobs/regenerate_post_counts_job_test.rb
new file mode 100644
index 000000000..ead222bb0
--- /dev/null
+++ b/test/jobs/regenerate_post_counts_job_test.rb
@@ -0,0 +1,18 @@
+require 'test_helper'
+
+class RegeneratePostCountsJobTest < ActiveJob::TestCase
+  context "RegeneratePostCountsJob" do
+    should "regenerate all incorrect tag post counts" do
+      tag1 = create(:tag, name: "touhou", post_count: -10)
+      tag2 = create(:tag, name: "bkub", post_count: 10)
+      tag3 = create(:tag, name: "chen", post_count: 10)
+      post = create(:post, tag_string: "touhou bkub")
+
+      RegeneratePostCountsJob.perform_now
+
+      assert_equal(1, Tag.find_by_name!("touhou").post_count)
+      assert_equal(1, Tag.find_by_name!("bkub").post_count)
+      assert_equal(0, Tag.find_by_name!("chen").post_count)
+    end
+  end
+end
diff --git a/test/unit/danbooru_maintenance_test.rb b/test/unit/danbooru_maintenance_test.rb
index 0155a77ff..5d73753ba 100644
--- a/test/unit/danbooru_maintenance_test.rb
+++ b/test/unit/danbooru_maintenance_test.rb
@@ -3,38 +3,36 @@ require 'test_helper'
 class DanbooruMaintenanceTest < ActiveSupport::TestCase
   context "hourly maintenance" do
     should "work" do
-      assert_nothing_raised { DanbooruMaintenance.hourly }
-    end
-
-    should "prune expired posts" do
-      @pending = create(:post, is_pending: true, created_at: 5.days.ago)
-      @flagged = create(:post, is_flagged: true, created_at: 5.days.ago)
-      @appealed = create(:post, is_deleted: true, created_at: 5.days.ago)
-
-      @flag = create(:post_flag, post: @flagged, created_at: 4.days.ago)
-      @appeal = create(:post_appeal, post: @appealed, created_at: 4.days.ago)
-
-      DanbooruMaintenance.hourly
-
-      assert_equal(true, @pending.reload.is_deleted?)
-      assert_equal(true, @flagged.reload.is_deleted?)
-      assert_equal(true, @appealed.reload.is_deleted?)
-      assert_equal(true, @flag.reload.succeeded?)
-      assert_equal(true, @appeal.reload.rejected?)
+      assert_nothing_raised do
+        DanbooruMaintenance.hourly
+        perform_enqueued_jobs
+      end
     end
   end
 
-  context "hourly maintenance" do
-    context "when pruning bans" do
-      should "clear the is_banned flag for users who are no longer banned" do
-        banner = FactoryBot.create(:admin_user)
-        user = FactoryBot.create(:user)
+  context "daily maintenance" do
+    should "work" do
+      assert_nothing_raised do
+        DanbooruMaintenance.daily
+        perform_enqueued_jobs
+      end
+    end
+  end
 
-        as(banner) { create(:ban, user: user, banner: banner, duration: 1) }
+  context "weekly maintenance" do
+    should "work" do
+      assert_nothing_raised do
+        DanbooruMaintenance.weekly
+        perform_enqueued_jobs
+      end
+    end
+  end
 
-        assert_equal(true, user.reload.is_banned)
-        travel_to(2.days.from_now) { DanbooruMaintenance.daily }
-        assert_equal(false, user.reload.is_banned)
+  context "monthly maintenance" do
+    should "work" do
+      assert_nothing_raised do
+        DanbooruMaintenance.monthly
+        perform_enqueued_jobs
       end
     end
   end
diff --git a/test/unit/tag_test.rb b/test/unit/tag_test.rb
index a638dc9a9..419181214 100644
--- a/test/unit/tag_test.rb
+++ b/test/unit/tag_test.rb
@@ -194,19 +194,4 @@ class TagTest < ActiveSupport::TestCase
       end
     end
   end
-
-  context "A tag with an incorrect post count" do
-    should "be fixed" do
-      tag1 = FactoryBot.create(:tag, name: "touhou", post_count: -10)
-      tag2 = FactoryBot.create(:tag, name: "bkub", post_count: 10)
-      tag3 = FactoryBot.create(:tag, name: "chen", post_count: 10)
-      post = FactoryBot.create(:post, tag_string: "touhou bkub")
-
-      tags = Tag.regenerate_post_counts!
-      assert_equal(3, tags.size)
-      assert_equal(1, tag1.reload.post_count)
-      assert_equal(1, tag2.reload.post_count)
-      assert_equal(0, tag3.reload.post_count)
-    end
-  end
 end