jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

additional tweaks to upload preprocessor

Browse Source
This commit is contained in:
Albert Yi
2018-06-19 10:51:05 -07:00
parent 88dec0f592
commit a64a9fa0d4
2 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/logical/upload_error_checker.rb
View File

@@ -4,7 +4,7 @@ require 'mail'
class UploadErrorChecker class UploadErrorChecker
def check! def check!
uploads = Upload.where("status like 'error%' and status not like 'error: RuntimeError - duplicate%' and created_at >= ?", 1.hour.ago) uploads = Upload.where("status like 'error%' and status not like 'error: Upload::Error - Post with MD5%' and status not like 'error: RuntimeError - duplicate%' and created_at >= ?", 1.hour.ago)
if uploads.size > 5 if uploads.size > 5
mail = Mail.new do mail = Mail.new do
from "webmaster@danbooru.donmai.us" from "webmaster@danbooru.donmai.us"

18
app/logical/upload_service.rb
View File

@@ -4,10 +4,6 @@ class UploadService
upload = Upload.new upload = Upload.new
if url if url
# this gets called from UploadsController#new so we need
# to preprocess async
Preprocessor.new(source: url).delay(queue: "default").start!(CurrentUser.user.id)
download = Downloads::File.new(url) download = Downloads::File.new(url)
normalized_url, _, _ = download.before_download(url, {}) normalized_url, _, _ = download.before_download(url, {})
post = if normalized_url.nil? post = if normalized_url.nil?
@@ -16,6 +12,12 @@ class UploadService
Post.where("SourcePattern(lower(posts.source)) IN (?)", [url, normalized_url]).first Post.where("SourcePattern(lower(posts.source)) IN (?)", [url, normalized_url]).first
end end
if post.nil?
# this gets called from UploadsController#new so we need
# to preprocess async
Preprocessor.new(source: url).delay(queue: "default").start!(CurrentUser.user.id)
end
begin begin
source = Sources::Site.new(url, :referer_url => ref) source = Sources::Site.new(url, :referer_url => ref)
remote_size = download.size remote_size = download.size
@@ -62,11 +64,15 @@ class UploadService
end end
end end
def self.delete_file(md5, file_ext) def self.delete_file(md5, file_ext, upload_id = nil)
if Post.where(md5: md5).exists? if Post.where(md5: md5).exists?
return return
end end
if upload_id
Upload.find(upload_id).update(status: "preprocessed + deleted")
end
Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :original) Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :original)
Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :large) Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :large)
Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :preview) Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :preview)
@@ -178,7 +184,7 @@ class UploadService
# in case this upload never finishes processing, we need to delete the # in case this upload never finishes processing, we need to delete the
# distributed files in the future # distributed files in the future
Danbooru.config.other_server_hosts.each do |host| Danbooru.config.other_server_hosts.each do |host|
UploadService::Utils.delay(queue: host, run_at: 10.minutes.from_now).delete_file(upload.md5, upload.file_ext) UploadService::Utils.delay(queue: host, run_at: 10.minutes.from_now).delete_file(upload.md5, upload.file_ext, upload.id)
end end
end end