jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor sources

Browse Source
This commit is contained in:
Albert Yi
2018-08-06 17:39:25 -07:00
parent 54363ffecf
commit 762dc3da24
71 changed files with 2340 additions and 2430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
app/logical/upload_service/controller_helper.rb
View File

@@ -4,13 +4,8 @@ class UploadService
upload = Upload.new
if Utils.is_downloadable?(url) && file.nil?
download = Downloads::File.new(url)
normalized_url = download.rewrite_url()
post = if normalized_url.nil?
Post.where("SourcePattern(lower(posts.source)) = ?", url).first
else
Post.where("SourcePattern(lower(posts.source)) IN (?)", [url, normalized_url]).first
end
strategy = Sources::Strategies.find(url, ref)
post = Post.where("SourcePattern(lower(posts.source)) IN (?)", [url, strategy.canonical_url]).first
if post.nil?
# this gets called from UploadsController#new so we need
@@ -19,13 +14,15 @@ class UploadService
end
begin
source = Sources::Site.new(url, :referer_url => ref)
download = Downloads::File.new(url, ref)
remote_size = download.size
rescue Exception
end
return [upload, post, source, normalized_url, remote_size]
elsif file
return [upload, post, strategy, remote_size]
end
if file
# this gets called via XHR so we can process sync
Preprocessor.new(file: file).delayed_start(CurrentUser.id)
end
@@ -35,9 +32,7 @@ class UploadService
def self.batch(url, ref = nil)
if url
source = Sources::Site.new(url, :referer_url => ref)
source.get
return source
return Sources::Strategies.find(url, ref)
end
end
end

65
app/logical/upload_service/preprocessor.rb
View File

@@ -1,5 +1,7 @@
class UploadService
class Preprocessor
extend Memoist
attr_reader :params, :original_post_id
def initialize(params)
@@ -15,31 +17,40 @@ class UploadService
params[:md5_confirmation]
end
def referer
def referer_url
params[:referer_url]
end
def normalized_source
@normalized_source ||= begin
Downloads::File.new(params[:source]).rewrite_url
end
def strategy
Sources::Strategies.find(source, referer_url)
end
memoize :strategy
# When searching posts we have to use the canonical source
def canonical_source
strategy.canonical_url
end
memoize :canonical_source
def in_progress?
if Utils.is_downloadable?(source)
Upload.where(status: "preprocessing", source: normalized_source).or(Upload.where(status: "preprocessing", alt_source: normalized_source)).exists?
elsif md5.present?
Upload.where(status: "preprocessing", md5: md5).exists?
else
false
return Upload.where(status: "preprocessing", source: source).exists?
end
if md5.present?
return Upload.where(status: "preprocessing", md5: md5).exists?
end
false
end
def predecessor
if Utils.is_downloadable?(source)
Upload.where(status: ["preprocessed", "preprocessing"]).where(source: normalized_source).or(Upload.where(status: ["preprocessed", "preprocessing"], alt_source: normalized_source)).first
elsif md5.present?
Upload.where(status: ["preprocessed", "preprocessing"], md5: md5).first
return Upload.where(status: ["preprocessed", "preprocessing"], source: source).first
end
if md5.present?
return Upload.where(status: ["preprocessed", "preprocessing"], md5: md5).first
end
end
@@ -59,34 +70,31 @@ class UploadService
def start!
if Utils.is_downloadable?(source)
CurrentUser.as_system do
if Post.tag_match("source:#{normalized_source}").where.not(id: original_post_id).exists?
raise ActiveRecord::RecordNotUnique.new("A post with source #{normalized_source} already exists")
if Post.tag_match("source:#{canonical_source}").where.not(id: original_post_id).exists?
raise ActiveRecord::RecordNotUnique.new("A post with source #{canonical_source} already exists")
end
end
if Upload.where(source: normalized_source, status: "completed").exists?
raise ActiveRecord::RecordNotUnique.new("A completed upload with source #{normalized_source} already exists")
if Upload.where(source: source, status: "completed").exists?
raise ActiveRecord::RecordNotUnique.new("A completed upload with source #{source} already exists")
end
if Upload.where(source: normalized_source).where("status like ?", "error%").exists?
raise ActiveRecord::RecordNotUnique.new("An errored upload with source #{normalized_source} already exists")
if Upload.where(source: source).where("status like ?", "error%").exists?
raise ActiveRecord::RecordNotUnique.new("An errored upload with source #{source} already exists")
end
end
params[:rating] ||= "q"
params[:tag_string] ||= "tagme"
upload = Upload.create!(params)
begin
upload.update(status: "preprocessing")
if Utils.is_downloadable?(source)
# preserve the original source (for twitter, the twimg:orig
# source, while the status url is stored in upload.source)
upload.alt_source = normalized_source
file = Utils.download_for_upload(source, upload)
elsif params[:file].present?
if params[:file].present?
file = params[:file]
elsif Utils.is_downloadable?(source)
file = Utils.download_for_upload(upload)
end
Utils.process_file(upload, file, original_post_id: original_post_id)
@@ -109,10 +117,7 @@ class UploadService
# goto whoever submitted the form
pred.initialize_attributes
# we went through a lot of trouble normalizing the source,
# so don't overwrite it with whatever the user provided
pred.source = "" if pred.source.nil?
pred.attributes = self.params.except(:source)
pred.attributes = self.params
# if a file was uploaded after the preprocessing occurred,
# then process the file and overwrite whatever the preprocessor

6
app/logical/upload_service/replacer.rb
View File

@@ -74,8 +74,8 @@ class UploadService
if replacement.replacement_file.present?
replacement.replacement_url = "file://#{replacement.replacement_file.original_filename}"
elsif upload.downloaded_source.present?
replacement.replacement_url = upload.downloaded_source
elsif upload.source.present?
replacement.replacement_url = Sources::Strategies.canonical(upload.source, upload.referer_url)
end
if md5_changed
@@ -93,7 +93,7 @@ class UploadService
post.image_width = upload.image_width
post.image_height = upload.image_height
post.file_size = upload.file_size
post.source = upload.downloaded_source || upload.source
post.source = Sources::Strategies.canonical(upload.source, upload.referer_url)
post.tag_string = upload.tag_string
update_ugoira_frame_data(post, upload)

36
app/logical/upload_service/utils.rb
View File

@@ -200,37 +200,19 @@ class UploadService
tags.join(" ")
end
def download_from_source(source, referer_url: nil)
download = Downloads::File.new(source, referer_url: referer_url)
file = download.download!
context = {
downloaded_source: download.downloaded_source,
source: download.source
}
def download_for_upload(upload)
download = Downloads::File.new(upload.source, upload.referer_url)
file, strategy = download.download!
if download.data[:is_ugoira]
context[:ugoira] = {
frame_data: download.data[:ugoira_frame_data],
content_type: download.data[:ugoira_content_type]
if download.data[:ugoira_frame_data]
upload.context = {
"ugoira" => {
"frame_data" => download.data[:ugoira_frame_data],
"content_type" => "image/jpeg"
}
}
end
yield(context)
return file
end
def download_for_upload(source, upload)
file = download_from_source(source, referer_url: upload.referer_url) do |context|
upload.downloaded_source = context[:downloaded_source]
upload.source = context[:source]
if context[:ugoira]
upload.context = { ugoira: context[:ugoira] }
end
end
return file
end
end