From f4953549ae450a94dd5ccf707fe9c5227c4a3ca2 Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 4 Jan 2022 13:52:08 -0600 Subject: [PATCH] jobs: switch from DelayedJob to GoodJob. Switch the ActiveJob backend from DelayedJob to GoodJob. Differences: * The job worker is run with `bin/good_job start` instead of `bin/delayed_job`. * Jobs have an 8 hour timeout instead of a 4 hour timeout. * Jobs don't automatically retry on failure. * Finishing jobs are preserved and pruned after 7 days. --- Procfile | 2 +- app/jobs/README.md | 26 ++++++++---------- app/jobs/application_job.rb | 8 ++++-- app/jobs/bigquery_export_job.rb | 7 ----- app/jobs/discord_notification_job.rb | 7 ----- app/jobs/process_bulk_update_request_job.rb | 7 ----- app/jobs/prune_delayed_jobs_job.rb | 9 ------- app/jobs/prune_jobs_job.rb | 8 ++++++ app/logical/danbooru_maintenance.rb | 2 +- bin/good_job | 29 +++++++++++++++++++++ config/environments/production.rb | 2 +- config/initializers/good_job.rb | 5 ++++ db/populate.rb | 1 - docker-compose.yaml | 4 +-- lib/tasks/danbooru.rake | 4 +-- 15 files changed, 66 insertions(+), 55 deletions(-) delete mode 100644 app/jobs/prune_delayed_jobs_job.rb create mode 100644 app/jobs/prune_jobs_job.rb create mode 100755 bin/good_job create mode 100644 config/initializers/good_job.rb diff --git a/Procfile b/Procfile index 084a01750..28ec3d887 100644 --- a/Procfile +++ b/Procfile @@ -8,7 +8,7 @@ # https://github.com/ddollar/foreman web: bin/rails server -worker: bin/rails jobs:work +worker: bin/good_job start clock: bin/rails danbooru:cron webpack-dev-server: bin/webpack-dev-server # db: docker run --rm -it --name danbooru-postgres --shm-size=8g -p 5432:5432 -e POSTGRES_USER=danbooru - e POSTRES_HOST_AUTH_METHOD=trust -v danbooru-postgres:/var/lib/postgresql/data ghcr.io/danbooru/postgres:14.0 diff --git a/app/jobs/README.md b/app/jobs/README.md index 4dd653fa5..4712d1d62 100644 --- a/app/jobs/README.md +++ b/app/jobs/README.md @@ -9,30 +9,26 @@ later. Jobs use the Rails Active Job framework. Active Job is a common framework that allows jobs to be run on different job runner backends. -In the production environment, jobs are run using the Delayed Job backend. Jobs -are stored in the database in the `delayed_job` table. Worker processes spawned -by `bin/delayed_job` poll the table for new jobs to work. +In the production environment, jobs are run using the Good Job backend. Jobs +are stored in the database in the `good_jobs` table. Worker processes spawned +by `bin/good_job` poll the table for new jobs to work. In the development environment, jobs are run with an in-process thread pool. This will run jobs in the background, but will drop jobs when the server is restarted. -There are two job queues, the `default` queue and the `bulk_update`. The -`bulk_update` queue handles bulk update requests. It has only one worker so that -bulk update requests are effectively processed sequentially. The `default` queue -handles everything else. - -There is a very minimal admin dashboard for jobs at https://danbooru.donmai.us/delayed_jobs. +There is a very minimal admin dashboard for jobs at https://danbooru.donmai.us/jobs. Danbooru also has periodic maintenance tasks that run in the background as cron -jobs. These are different from the jobs in this directory. See [app/logical/danbooru_maintenance.rb](../logical/danbooru_maintenance.rb). +jobs. These are different from the jobs in this directory. See +[app/logical/danbooru_maintenance.rb](../logical/danbooru_maintenance.rb). # Usage Start a pool of job workers: ``` -RAILS_ENV=production bin/delayed_job --pool=default:8 --pool=bulk_update start +RAILS_ENV=production bin/good_job start --max-threads=4 ``` # Examples @@ -47,12 +43,12 @@ DeleteFavoritesJob.perform_later(user) # See also * [app/logical/danbooru_maintenance.rb](../logical/danbooru_maintenance.rb) -* [app/controllers/delayed_jobs_controller.rb](../controllers/delayed_jobs_controller.rb) -* [config/initializers/delayed_jobs.rb](../../config/initializers/delayed_jobs.rb) +* [app/controllers/jobs_controller.rb](../controllers/jobs_controller.rb) +* [config/initializers/good_job.rb](../../config/initializers/good_job.rb) * [test/jobs](../../test/jobs) # External links * https://guides.rubyonrails.org/active_job_basics.html -* https://github.com/collectiveidea/delayed_job -* https://danbooru.donmai.us/delayed_jobs \ No newline at end of file +* https://github.com/bensheldon/good_job +* https://danbooru.donmai.us/jobs diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 1781f55eb..50acb6a4b 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -3,15 +3,19 @@ # The base class for all background jobs on Danbooru. # # @see https://guides.rubyonrails.org/active_job_basics.html -# @see https://github.com/collectiveidea/delayed_job +# @see https://github.com/bensheldon/good_job class ApplicationJob < ActiveJob::Base + class JobTimeoutError < StandardError; end + queue_as :default queue_with_priority 0 around_perform do |_job, block| CurrentUser.scoped(User.system, "127.0.0.1") do ApplicationRecord.without_timeout do - block.call + Timeout.timeout(8.hours, JobTimeoutError) do + block.call + end end end end diff --git a/app/jobs/bigquery_export_job.rb b/app/jobs/bigquery_export_job.rb index e14a6db6e..7e73dadf0 100644 --- a/app/jobs/bigquery_export_job.rb +++ b/app/jobs/bigquery_export_job.rb @@ -5,13 +5,6 @@ # # @see BigqueryExportService class BigqueryExportJob < ApplicationJob - retry_on Exception, attempts: 0 - - # XXX delayed_job specific - def max_attempts - 1 - end - def perform(model:, **options) BigqueryExportService.new(model, **options).export! end diff --git a/app/jobs/discord_notification_job.rb b/app/jobs/discord_notification_job.rb index 56c3ef584..cf9e39fd0 100644 --- a/app/jobs/discord_notification_job.rb +++ b/app/jobs/discord_notification_job.rb @@ -3,13 +3,6 @@ # A job that sends notifications about new forum posts to Discord. Spawned by # the {ForumPost} class when a new forum post is created. class DiscordNotificationJob < ApplicationJob - retry_on Exception, attempts: 0 - - # XXX delayed_job specific - def max_attempts - 1 - end - def perform(forum_post:) forum_post.send_discord_notification end diff --git a/app/jobs/process_bulk_update_request_job.rb b/app/jobs/process_bulk_update_request_job.rb index 93fde017e..1801a45db 100644 --- a/app/jobs/process_bulk_update_request_job.rb +++ b/app/jobs/process_bulk_update_request_job.rb @@ -5,13 +5,6 @@ # @see {BulkUpdateRequestProcessor} # @see {BulkUpdateRequest} class ProcessBulkUpdateRequestJob < ApplicationJob - retry_on Exception, attempts: 0 - - # XXX delayed_job specific - def max_attempts - 1 - end - def perform(bulk_update_request) bulk_update_request.processor.process! end diff --git a/app/jobs/prune_delayed_jobs_job.rb b/app/jobs/prune_delayed_jobs_job.rb deleted file mode 100644 index 2474c6589..000000000 --- a/app/jobs/prune_delayed_jobs_job.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -# A job that runs daily to delete all stale delayed jobs. Spawned by -# {DanbooruMaintenance}. -class PruneDelayedJobsJob < ApplicationJob - def perform - Delayed::Job.where("created_at < ?", 45.days.ago).delete_all - end -end diff --git a/app/jobs/prune_jobs_job.rb b/app/jobs/prune_jobs_job.rb new file mode 100644 index 000000000..ab67ce95b --- /dev/null +++ b/app/jobs/prune_jobs_job.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# A job that runs daily to delete all stale jobs. Spawned by {DanbooruMaintenance}. +class PruneJobsJob < ApplicationJob + def perform + GoodJob::ActiveJobJob.where("created_at < ?", 7.days.ago).destroy_all + end +end diff --git a/app/logical/danbooru_maintenance.rb b/app/logical/danbooru_maintenance.rb index 2a399e67c..51fa99911 100644 --- a/app/logical/danbooru_maintenance.rb +++ b/app/logical/danbooru_maintenance.rb @@ -12,7 +12,7 @@ module DanbooruMaintenance end def daily - queue PruneDelayedJobsJob + queue PruneJobsJob queue PrunePostDisapprovalsJob queue PruneBulkUpdateRequestsJob queue PruneBansJob diff --git a/bin/good_job b/bin/good_job new file mode 100755 index 000000000..fe329912b --- /dev/null +++ b/bin/good_job @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# +# This file was generated by Bundler. +# +# The application 'good_job' is installed as part of a gem, and +# this file is here to facilitate running it. +# + +require "pathname" +ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", + Pathname.new(__FILE__).realpath) + +bundle_binstub = File.expand_path("../bundle", __FILE__) + +if File.file?(bundle_binstub) + if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ + load(bundle_binstub) + else + abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. +Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") + end +end + +require "rubygems" +require "bundler/setup" + +load Gem.bin_path("good_job", "good_job") diff --git a/config/environments/production.rb b/config/environments/production.rb index dbabb916f..b1645ce7b 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -45,7 +45,7 @@ Rails.application.configure do # config.cache_store = :mem_cache_store # Use a real queuing backend for Active Job (and separate queues per environment). - config.active_job.queue_adapter = :delayed_job + config.active_job.queue_adapter = :good_job # config.active_job.queue_name_prefix = "danbooru_production" config.action_mailer.perform_caching = false diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb new file mode 100644 index 000000000..921f75e5f --- /dev/null +++ b/config/initializers/good_job.rb @@ -0,0 +1,5 @@ +GoodJob.retry_on_unhandled_error = false +GoodJob.preserve_job_records = true +GoodJob.on_thread_error = ->(exception) do + DanbooruLogger.log(exception) +end diff --git a/db/populate.rb b/db/populate.rb index 4195cc6cf..b9b4addf5 100644 --- a/db/populate.rb +++ b/db/populate.rb @@ -1,7 +1,6 @@ require 'set' CurrentUser.ip_addr = "127.0.0.1" -Delayed::Worker.delay_jobs = false used_names = Set.new([""]) def rand_string(n, unique = false) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1ec736f4a..53cad39db 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -56,7 +56,7 @@ services: - "danbooru-images:/danbooru/public/data" command: ["bash", "-c", "bin/wait-for-http http://danbooru:3000 5s && bin/rails danbooru:cron"] - delayed_jobs: + jobs: # We need root to write temp upload files in the images directory (/danbooru/public/data) user: root image: ghcr.io/danbooru/danbooru:production @@ -71,7 +71,7 @@ services: volumes: # We need access to images so we can add/remove images to IQDB. - "danbooru-images:/danbooru/public/data" - command: ["bash", "-c", "bin/wait-for-http http://danbooru:3000 5s && bin/delayed_job run"] + command: ["bash", "-c", "bin/wait-for-http http://danbooru:3000 5s && bin/good_job start"] # https://github.com/danbooru/iqdb # https://hub.docker.com/repository/docker/evazion/iqdb diff --git a/lib/tasks/danbooru.rake b/lib/tasks/danbooru.rake index 784249e70..c425bb23a 100644 --- a/lib/tasks/danbooru.rake +++ b/lib/tasks/danbooru.rake @@ -15,8 +15,8 @@ namespace :danbooru do # Usage: bin/rails danbooru:reindex_iqdb # - # Schedules all posts to be reindexed in IQDB. Requires the delayed jobs - # worker (bin/delayed_job) to be running. + # Schedules all posts to be reindexed in IQDB. Requires the jobs + # worker (bin/good_job) to be running. desc "Reindex all posts in IQDB" task reindex_iqdb: :environment do Post.find_each do |post|