diff --git a/app/jobs/bigquery_export_all_job.rb b/app/jobs/bigquery_export_all_job.rb new file mode 100644 index 000000000..195734ff0 --- /dev/null +++ b/app/jobs/bigquery_export_all_job.rb @@ -0,0 +1,6 @@ +# A job that runs daily to export all tables to BigQuery. Spawned by {DanbooruMaintenance}. +class BigqueryExportAllJob < ApplicationJob + def perform + BigqueryExportService.async_export_all! + end +end diff --git a/app/jobs/dmail_inactive_approvers_job.rb b/app/jobs/dmail_inactive_approvers_job.rb new file mode 100644 index 000000000..588ef2ecd --- /dev/null +++ b/app/jobs/dmail_inactive_approvers_job.rb @@ -0,0 +1,7 @@ +# A job that runs weekly to warn inactive approvers before they're demoted. +# Spawned by {DanbooruMaintenance}. +class DmailInactiveApproversJob < ApplicationJob + def perform + ApproverPruner.dmail_inactive_approvers! + end +end diff --git a/app/jobs/prune_approvers_job.rb b/app/jobs/prune_approvers_job.rb new file mode 100644 index 000000000..5dcc80411 --- /dev/null +++ b/app/jobs/prune_approvers_job.rb @@ -0,0 +1,7 @@ +# A job that runs monthly to demote all inactive approvers. Spawned by +# {DanbooruMaintenance}. +class PruneApproversJob < ApplicationJob + def perform + ApproverPruner.prune! + end +end diff --git a/app/jobs/prune_bans_job.rb b/app/jobs/prune_bans_job.rb new file mode 100644 index 000000000..952ea96bd --- /dev/null +++ b/app/jobs/prune_bans_job.rb @@ -0,0 +1,7 @@ +# A job that runs daily to remove expired bans. Spawned by +# {DanbooruMaintenance}. +class PruneBansJob < ApplicationJob + def perform + Ban.prune! + end +end diff --git a/app/jobs/prune_bulk_update_requests_job.rb b/app/jobs/prune_bulk_update_requests_job.rb new file mode 100644 index 000000000..5d6fdfb39 --- /dev/null +++ b/app/jobs/prune_bulk_update_requests_job.rb @@ -0,0 +1,8 @@ +# A job that runs daily to reject expired bulk update requests. Spawned by +# {DanbooruMaintenance}. +class PruneBulkUpdateRequestsJob < ApplicationJob + def perform + BulkUpdateRequestPruner.warn_old + BulkUpdateRequestPruner.reject_expired + end +end diff --git a/app/jobs/prune_delayed_jobs_job.rb b/app/jobs/prune_delayed_jobs_job.rb new file mode 100644 index 000000000..caa986b6f --- /dev/null +++ b/app/jobs/prune_delayed_jobs_job.rb @@ -0,0 +1,7 @@ +# A job that runs daily to delete all stale delayed jobs. Spawned by +# {DanbooruMaintenance}. +class PruneDelayedJobsJob < ApplicationJob + def perform + Delayed::Job.where("created_at < ?", 45.days.ago).delete_all + end +end diff --git a/app/jobs/prune_post_disapprovals_job.rb b/app/jobs/prune_post_disapprovals_job.rb new file mode 100644 index 000000000..f26f12e6a --- /dev/null +++ b/app/jobs/prune_post_disapprovals_job.rb @@ -0,0 +1,7 @@ +# A job that runs daily to remove old post disapprovals. Spawned by +# {DanbooruMaintenance}. +class PrunePostDisapprovalsJob < ApplicationJob + def perform + PostDisapproval.prune! + end +end diff --git a/app/jobs/prune_posts_job.rb b/app/jobs/prune_posts_job.rb new file mode 100644 index 000000000..2fcab7739 --- /dev/null +++ b/app/jobs/prune_posts_job.rb @@ -0,0 +1,7 @@ +# A job that runs hourly to delete all expired pending, flagged, and appealed +# posts. Spawned by {DanbooruMaintenance}. +class PrunePostsJob < ApplicationJob + def perform + PostPruner.prune! + end +end diff --git a/app/jobs/prune_rate_limits_job.rb b/app/jobs/prune_rate_limits_job.rb new file mode 100644 index 000000000..d9291d981 --- /dev/null +++ b/app/jobs/prune_rate_limits_job.rb @@ -0,0 +1,7 @@ +# A job that runs hourly to delete all state rate limit objects from the +# database. Spawned by {DanbooruMaintenance}. +class PruneRateLimitsJob < ApplicationJob + def perform + RateLimit.prune! + end +end diff --git a/app/jobs/prune_uploads_job.rb b/app/jobs/prune_uploads_job.rb new file mode 100644 index 000000000..378abf2a6 --- /dev/null +++ b/app/jobs/prune_uploads_job.rb @@ -0,0 +1,7 @@ +# A job that runs hourly to delete all completed, stale, or failed uploads. +# Spawned by {DanbooruMaintenance}. +class PruneUploadsJob < ApplicationJob + def perform + Upload.prune! + end +end diff --git a/app/jobs/regenerate_post_counts_job.rb b/app/jobs/regenerate_post_counts_job.rb new file mode 100644 index 000000000..0cd7e9bc0 --- /dev/null +++ b/app/jobs/regenerate_post_counts_job.rb @@ -0,0 +1,10 @@ +# A job that runs hourly to fix all incorrect tag counts. +# Spawned by {DanbooruMaintenance}. +class RegeneratePostCountsJob < ApplicationJob + def perform + updated_tags = Tag.regenerate_post_counts! + updated_tags.each do |tag| + DanbooruLogger.info("Updated tag count", tag.attributes) + end + end +end diff --git a/app/jobs/retire_tag_relationships_job.rb b/app/jobs/retire_tag_relationships_job.rb new file mode 100644 index 000000000..7b2e64f7f --- /dev/null +++ b/app/jobs/retire_tag_relationships_job.rb @@ -0,0 +1,7 @@ +# A job that runs weekly to retire inactive aliases and implications. Spawned +# by {DanbooruMaintenance}. +class RetireTagRelationshipsJob < ApplicationJob + def perform + TagRelationshipRetirementService.find_and_retire! + end +end diff --git a/app/jobs/vacuum_database_job.rb b/app/jobs/vacuum_database_job.rb new file mode 100644 index 000000000..34eb521b1 --- /dev/null +++ b/app/jobs/vacuum_database_job.rb @@ -0,0 +1,8 @@ +# A job that runs daily to vacuum the database. Spawned by {DanbooruMaintenance}. +class VacuumDatabaseJob < ApplicationJob + def perform + # We can't perform vacuum inside a transaction. This happens during tests. + return if ApplicationRecord.connection.transaction_open? + ApplicationRecord.connection.execute("vacuum analyze") + end +end diff --git a/app/logical/danbooru_maintenance.rb b/app/logical/danbooru_maintenance.rb index 19c5cd841..58f36187b 100644 --- a/app/logical/danbooru_maintenance.rb +++ b/app/logical/danbooru_maintenance.rb @@ -2,45 +2,34 @@ module DanbooruMaintenance module_function def hourly - safely { Upload.prune! } - safely { PostPruner.prune! } - safely { RateLimit.prune! } - safely { regenerate_post_counts! } + queue PruneUploadsJob + queue PrunePostsJob + queue PruneRateLimitsJob + queue RegeneratePostCountsJob end def daily - safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all } - safely { PostDisapproval.prune! } - safely { BulkUpdateRequestPruner.warn_old } - safely { BulkUpdateRequestPruner.reject_expired } - safely { Ban.prune! } - safely { BigqueryExportService.async_export_all! } - safely { ActiveRecord::Base.connection.execute("vacuum analyze") unless Rails.env.test? } + queue PruneDelayedJobsJob + queue PrunePostDisapprovalsJob + queue PruneBulkUpdateRequestsJob + queue PruneBansJob + queue BigqueryExportAllJob + queue VacuumDatabaseJob end def weekly - safely { TagRelationshipRetirementService.find_and_retire! } - safely { ApproverPruner.dmail_inactive_approvers! } + queue RetireTagRelationshipsJob + queue DmailInactiveApproversJob end def monthly - safely { ApproverPruner.prune! } + queue PruneApproversJob end - def regenerate_post_counts! - updated_tags = Tag.regenerate_post_counts! - updated_tags.each do |tag| - DanbooruLogger.info("Updated tag count", tag.attributes) - end - end - - def safely(&block) - ActiveRecord::Base.connection.execute("set statement_timeout = 0") - - CurrentUser.scoped(User.system, "127.0.0.1") do - yield - end - rescue StandardError => exception + def queue(job) + DanbooruLogger.info("Queueing #{job.name}") + job.perform_later + rescue Exception # rubocop:disable Lint/RescueException DanbooruLogger.log(exception) raise exception if Rails.env.test? end diff --git a/test/jobs/prune_bans_job_test.rb b/test/jobs/prune_bans_job_test.rb new file mode 100644 index 000000000..7a456b55a --- /dev/null +++ b/test/jobs/prune_bans_job_test.rb @@ -0,0 +1,18 @@ +require 'test_helper' + +class PruneBansJobTest < ActiveJob::TestCase + context "PruneBansJob" do + should "prune all expired bans" do + @expired_ban = travel_to(1.month.ago) { create(:ban, duration: 1.week) } + @unexpired_ban = create(:ban, duration: 1.week) + + assert_equal(true, @expired_ban.user.is_banned?) + assert_equal(true, @unexpired_ban.user.is_banned?) + + PruneBansJob.perform_now + + assert_equal(false, @expired_ban.user.reload.is_banned?) + assert_equal(true, @unexpired_ban.user.reload.is_banned?) + end + end +end diff --git a/test/jobs/prune_posts_job_test.rb b/test/jobs/prune_posts_job_test.rb new file mode 100644 index 000000000..ce568c924 --- /dev/null +++ b/test/jobs/prune_posts_job_test.rb @@ -0,0 +1,22 @@ +require 'test_helper' + +class PrunePostsJobTest < ActiveJob::TestCase + context "PrunePostsJob" do + should "prune expired posts" do + @pending = create(:post, is_pending: true, created_at: 5.days.ago) + @flagged = create(:post, is_flagged: true, created_at: 5.days.ago) + @appealed = create(:post, is_deleted: true, created_at: 5.days.ago) + + @flag = create(:post_flag, post: @flagged, created_at: 4.days.ago) + @appeal = create(:post_appeal, post: @appealed, created_at: 4.days.ago) + + PrunePostsJob.perform_now + + assert_equal(true, @pending.reload.is_deleted?) + assert_equal(true, @flagged.reload.is_deleted?) + assert_equal(true, @appealed.reload.is_deleted?) + assert_equal(true, @flag.reload.succeeded?) + assert_equal(true, @appeal.reload.rejected?) + end + end +end diff --git a/test/jobs/prune_rate_limits_job_test.rb b/test/jobs/prune_rate_limits_job_test.rb new file mode 100644 index 000000000..8868f242d --- /dev/null +++ b/test/jobs/prune_rate_limits_job_test.rb @@ -0,0 +1,13 @@ +require 'test_helper' + +class PruneRateLimitsJobTest < ActiveJob::TestCase + context "PruneRateLimitsJob" do + should "prune all stale rate limits" do + travel_to(2.hours.ago) { create(:rate_limit) } + + assert_equal(1, RateLimit.count) + PruneRateLimitsJob.perform_now + assert_equal(0, RateLimit.count) + end + end +end diff --git a/test/jobs/prune_uploads_job_test.rb b/test/jobs/prune_uploads_job_test.rb new file mode 100644 index 000000000..4446229f4 --- /dev/null +++ b/test/jobs/prune_uploads_job_test.rb @@ -0,0 +1,19 @@ +require 'test_helper' + +class PruneUploadsJobTest < ActiveJob::TestCase + context "PruneUploadsJob" do + should "prune all old uploads" do + @uploader = create(:user) + + as(@uploader) do + @completed_upload = travel_to(2.hours.ago) { create(:upload, uploader: @uploader, status: "completed") } + @stale_upload = travel_to(2.days.ago) { create(:upload, uploader: @uploader, status: "preprocessed") } + @failed_upload = travel_to(4.days.ago) { create(:upload, uploader: @uploader, status: "error") } + end + + assert_equal(3, Upload.count) + PruneUploadsJob.perform_now + assert_equal(0, Upload.count) + end + end +end diff --git a/test/jobs/regenerate_post_counts_job_test.rb b/test/jobs/regenerate_post_counts_job_test.rb new file mode 100644 index 000000000..ead222bb0 --- /dev/null +++ b/test/jobs/regenerate_post_counts_job_test.rb @@ -0,0 +1,18 @@ +require 'test_helper' + +class RegeneratePostCountsJobTest < ActiveJob::TestCase + context "RegeneratePostCountsJob" do + should "regenerate all incorrect tag post counts" do + tag1 = create(:tag, name: "touhou", post_count: -10) + tag2 = create(:tag, name: "bkub", post_count: 10) + tag3 = create(:tag, name: "chen", post_count: 10) + post = create(:post, tag_string: "touhou bkub") + + RegeneratePostCountsJob.perform_now + + assert_equal(1, Tag.find_by_name!("touhou").post_count) + assert_equal(1, Tag.find_by_name!("bkub").post_count) + assert_equal(0, Tag.find_by_name!("chen").post_count) + end + end +end diff --git a/test/unit/danbooru_maintenance_test.rb b/test/unit/danbooru_maintenance_test.rb index 0155a77ff..5d73753ba 100644 --- a/test/unit/danbooru_maintenance_test.rb +++ b/test/unit/danbooru_maintenance_test.rb @@ -3,38 +3,36 @@ require 'test_helper' class DanbooruMaintenanceTest < ActiveSupport::TestCase context "hourly maintenance" do should "work" do - assert_nothing_raised { DanbooruMaintenance.hourly } - end - - should "prune expired posts" do - @pending = create(:post, is_pending: true, created_at: 5.days.ago) - @flagged = create(:post, is_flagged: true, created_at: 5.days.ago) - @appealed = create(:post, is_deleted: true, created_at: 5.days.ago) - - @flag = create(:post_flag, post: @flagged, created_at: 4.days.ago) - @appeal = create(:post_appeal, post: @appealed, created_at: 4.days.ago) - - DanbooruMaintenance.hourly - - assert_equal(true, @pending.reload.is_deleted?) - assert_equal(true, @flagged.reload.is_deleted?) - assert_equal(true, @appealed.reload.is_deleted?) - assert_equal(true, @flag.reload.succeeded?) - assert_equal(true, @appeal.reload.rejected?) + assert_nothing_raised do + DanbooruMaintenance.hourly + perform_enqueued_jobs + end end end - context "hourly maintenance" do - context "when pruning bans" do - should "clear the is_banned flag for users who are no longer banned" do - banner = FactoryBot.create(:admin_user) - user = FactoryBot.create(:user) + context "daily maintenance" do + should "work" do + assert_nothing_raised do + DanbooruMaintenance.daily + perform_enqueued_jobs + end + end + end - as(banner) { create(:ban, user: user, banner: banner, duration: 1) } + context "weekly maintenance" do + should "work" do + assert_nothing_raised do + DanbooruMaintenance.weekly + perform_enqueued_jobs + end + end + end - assert_equal(true, user.reload.is_banned) - travel_to(2.days.from_now) { DanbooruMaintenance.daily } - assert_equal(false, user.reload.is_banned) + context "monthly maintenance" do + should "work" do + assert_nothing_raised do + DanbooruMaintenance.monthly + perform_enqueued_jobs end end end diff --git a/test/unit/tag_test.rb b/test/unit/tag_test.rb index a638dc9a9..419181214 100644 --- a/test/unit/tag_test.rb +++ b/test/unit/tag_test.rb @@ -194,19 +194,4 @@ class TagTest < ActiveSupport::TestCase end end end - - context "A tag with an incorrect post count" do - should "be fixed" do - tag1 = FactoryBot.create(:tag, name: "touhou", post_count: -10) - tag2 = FactoryBot.create(:tag, name: "bkub", post_count: 10) - tag3 = FactoryBot.create(:tag, name: "chen", post_count: 10) - post = FactoryBot.create(:post, tag_string: "touhou bkub") - - tags = Tag.regenerate_post_counts! - assert_equal(3, tags.size) - assert_equal(1, tag1.reload.post_count) - assert_equal(1, tag2.reload.post_count) - assert_equal(0, tag3.reload.post_count) - end - end end