jobs: migrate file deletion jobs to ActiveJob.

This commit is contained in:
evazion
2019-08-16 20:49:35 -05:00
parent 2b6361369e
commit 868a2256d1
6 changed files with 34 additions and 7 deletions

View File

@@ -0,0 +1,8 @@
class DeletePostFilesJob < ApplicationJob
queue_as :default
queue_with_priority 20
def perform(id, md5, file_ext)
Post.delete_files(id, md5, file_ext)
end
end

View File

@@ -0,0 +1,8 @@
class DeleteUploadFilesJob < ApplicationJob
queue_as :default
queue_with_priority 20
def perform(md5, file_ext, upload_id)
UploadService::Utils.delete_file(md5, file_ext, upload_id)
end
end

View File

@@ -166,7 +166,7 @@ class UploadService
# in case this upload never finishes processing, we need to delete the # in case this upload never finishes processing, we need to delete the
# distributed files in the future # distributed files in the future
UploadService::Utils.delay(priority: -1, queue: "default", run_at: 24.hours.from_now).delete_file(upload.md5, upload.file_ext, upload.id) DeleteUploadFilesJob.set(wait: 24.hours).perform_later(upload.md5, upload.file_ext, upload.id)
end end
# these methods are only really used during upload processing even # these methods are only really used during upload processing even

View File

@@ -88,7 +88,7 @@ class Post < ApplicationRecord
end end
def queue_delete_files(grace_period) def queue_delete_files(grace_period)
Post.delay(queue: "default", run_at: Time.now + grace_period).delete_files(id, md5, file_ext) DeletePostFilesJob.set(wait: grace_period).perform_later(id, md5, file_ext)
end end
def delete_files def delete_files

View File

@@ -838,7 +838,7 @@ class UploadServiceTest < ActiveSupport::TestCase
as_user { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") } as_user { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") }
travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do
Delayed::Worker.new.work_off workoff_active_jobs
end end
rescue Net::OpenTimeout rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed" skip "Remote connection to Pixiv failed"
@@ -887,7 +887,7 @@ class UploadServiceTest < ActiveSupport::TestCase
assert_nothing_raised { @post.file(:preview) } assert_nothing_raised { @post.file(:preview) }
travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do
Delayed::Worker.new.work_off workoff_active_jobs
end end
assert_nothing_raised { @post.file(:original) } assert_nothing_raised { @post.file(:original) }
@@ -914,20 +914,24 @@ class UploadServiceTest < ActiveSupport::TestCase
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post1.md5) assert_equal("4ceadc314938bc27f3574053a3e1459a", @post1.md5)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post2.md5) assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post2.md5)
@post2.reload @post2.reload
@post2.replace!(replacement_url: "https://raikou1.donmai.us/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg") @post2.replace!(replacement_url: "https://raikou1.donmai.us/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg")
assert_equal("d34e4cf0a437a5d65f8e82b7bcd02606", @post2.md5) assert_equal("d34e4cf0a437a5d65f8e82b7bcd02606", @post2.md5)
Upload.destroy_all Upload.destroy_all
@post1.reload @post1.reload
@post2.reload @post2.reload
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post1.md5) assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post1.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post2.md5) assert_equal("4ceadc314938bc27f3574053a3e1459a", @post2.md5)
end end
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do travel_to (PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now do
Delayed::Worker.new.work_off assert_raise(Post::DeletionError) do
workoff_active_jobs
end
end end
assert_nothing_raised { @post1.file(:original) } assert_nothing_raised { @post1.file(:original) }

View File

@@ -53,8 +53,15 @@ module TestHelpers
Thread.current[:pixiv_comic_session_cache_key] = Cache.get(PixivWebAgent::COMIC_SESSION_CACHE_KEY) Thread.current[:pixiv_comic_session_cache_key] = Cache.get(PixivWebAgent::COMIC_SESSION_CACHE_KEY)
end end
end end
end
# XXX replace with `perform_enqueued_jobs` after rails 6 upgrade.
def workoff_active_jobs
queue_adapter.enqueued_jobs.each do |job_data|
klass = job_data[:job]
klass.perform_now(*job_data[:args])
end
end
end
class ActiveSupport::TestCase class ActiveSupport::TestCase
include ActiveJob::TestHelper include ActiveJob::TestHelper