add support for upload preprocessing
This commit is contained in:
Albert Yi
@@ -3,31 +3,14 @@ class UploadsController < ApplicationController
|
||||
respond_to :html, :xml, :json, :js
|
||||
|
||||
def new
|
||||
@upload = Upload.new
|
||||
@upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
|
||||
if params[:url]
|
||||
download = Downloads::File.new(params[:url])
|
||||
@normalized_url, _, _ = download.before_download(params[:url], {})
|
||||
@post = find_post_by_url(@normalized_url)
|
||||
|
||||
begin
|
||||
@source = Sources::Site.new(params[:url], :referer_url => params[:ref])
|
||||
@remote_size = download.size
|
||||
rescue Exception
|
||||
end
|
||||
end
|
||||
@upload, @post, @source, @normalized_url, @remote_size = UploadService::ControllerHelper.prepare(params[:url], params[:ref])
|
||||
respond_with(@upload)
|
||||
end
|
||||
|
||||
def batch
|
||||
@url = params.dig(:batch, :url) || params[:url]
|
||||
@source = nil
|
||||
|
||||
if @url
|
||||
@source = Sources::Site.new(@url, :referer_url => params[:ref])
|
||||
@source.get
|
||||
end
|
||||
|
||||
@source = UploadService::ControllerHelper.batch(@url, params[:ref])
|
||||
respond_with(@source)
|
||||
end
|
||||
|
||||
@@ -57,14 +40,11 @@ class UploadsController < ApplicationController
|
||||
end
|
||||
|
||||
def create
|
||||
@upload = Upload.create(upload_params)
|
||||
@service = UploadService.new(upload_params)
|
||||
@upload = @service.start!
|
||||
|
||||
if @upload.errors.empty?
|
||||
post = @upload.process!
|
||||
|
||||
if post.present? && post.valid? && post.warnings.any?
|
||||
flash[:notice] = post.warnings.full_messages.join(".\n \n")
|
||||
end
|
||||
if @service.warnings.any?
|
||||
flash[:notice] = @service.warnings.join(".\n \n")
|
||||
end
|
||||
|
||||
save_recent_tags
|
||||
@@ -73,14 +53,6 @@ class UploadsController < ApplicationController
|
||||
|
||||
private
|
||||
|
||||
def find_post_by_url(normalized_url)
|
||||
if normalized_url.nil?
|
||||
Post.where("SourcePattern(lower(posts.source)) = ?", params[:url]).first
|
||||
else
|
||||
Post.where("SourcePattern(lower(posts.source)) IN (?)", [params[:url], @normalized_url]).first
|
||||
end
|
||||
end
|
||||
|
||||
def save_recent_tags
|
||||
if @upload
|
||||
tags = Tag.scan_tags(@upload.tag_string)
|
||||
@@ -94,7 +66,7 @@ class UploadsController < ApplicationController
|
||||
permitted_params = %i[
|
||||
file source tag_string rating status parent_id artist_commentary_title
|
||||
artist_commentary_desc include_artist_commentary referer_url
|
||||
md5_confirmation as_pending
|
||||
md5_confirmation as_pending
|
||||
]
|
||||
|
||||
params.require(:upload).permit(permitted_params)
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
class PixivUgoiraService
|
||||
attr_reader :width, :height, :frame_data, :content_type
|
||||
|
||||
def save_frame_data(post)
|
||||
PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id)
|
||||
end
|
||||
|
||||
def calculate_dimensions(source_path)
|
||||
folder = Zip::File.new(source_path)
|
||||
tempfile = Tempfile.new("ugoira-dimensions")
|
||||
|
||||
begin
|
||||
folder.first.extract(tempfile.path) {true}
|
||||
image_size = ImageSpec.new(tempfile.path)
|
||||
@width = image_size.width
|
||||
@height = image_size.height
|
||||
ensure
|
||||
tempfile.close
|
||||
tempfile.unlink
|
||||
end
|
||||
end
|
||||
|
||||
def load(data)
|
||||
if data[:is_ugoira]
|
||||
@frame_data = data[:ugoira_frame_data]
|
||||
@content_type = data[:ugoira_content_type]
|
||||
end
|
||||
end
|
||||
|
||||
def empty?
|
||||
@frame_data.nil?
|
||||
end
|
||||
end
|
||||
399
app/logical/upload_service.rb
Normal file
399
app/logical/upload_service.rb
Normal file
@@ -0,0 +1,399 @@
|
||||
class UploadService
|
||||
module ControllerHelper
|
||||
def self.prepare(url, ref = nil)
|
||||
upload = Upload.new
|
||||
|
||||
if url
|
||||
Preprocessor.new(source: url).delay(queue: "default").start!(CurrentUser.user.id)
|
||||
|
||||
download = Downloads::File.new(url)
|
||||
normalized_url, _, _ = download.before_download(url, {})
|
||||
post = if normalized_url.nil?
|
||||
Post.where("SourcePattern(lower(posts.source)) = ?", url).first
|
||||
else
|
||||
Post.where("SourcePattern(lower(posts.source)) IN (?)", [url, normalized_url]).first
|
||||
end
|
||||
|
||||
begin
|
||||
source = Sources::Site.new(url, :referer_url => ref)
|
||||
remote_size = download.size
|
||||
rescue Exception
|
||||
end
|
||||
|
||||
return [upload, post, source, normalized_url, remote_size]
|
||||
end
|
||||
|
||||
return [upload]
|
||||
end
|
||||
|
||||
def self.batch(url, ref = nil)
|
||||
if url
|
||||
source = Sources::Site.new(url, :referer_url => ref)
|
||||
source.get
|
||||
return source
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module Utils
|
||||
def self.file_header_to_file_ext(file)
|
||||
case File.read(file.path, 16)
|
||||
when /^\xff\xd8/n
|
||||
"jpg"
|
||||
when /^GIF87a/, /^GIF89a/
|
||||
"gif"
|
||||
when /^\x89PNG\r\n\x1a\n/n
|
||||
"png"
|
||||
when /^CWS/, /^FWS/, /^ZWS/
|
||||
"swf"
|
||||
when /^\x1a\x45\xdf\xa3/n
|
||||
"webm"
|
||||
when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
|
||||
"mp4"
|
||||
when /^PK\x03\x04/
|
||||
"zip"
|
||||
else
|
||||
"bin"
|
||||
end
|
||||
end
|
||||
|
||||
def self.calculate_ugoira_dimensions(source_path)
|
||||
folder = Zip::File.new(source_path)
|
||||
Tempfile.open("ugoira-dim-") do |tempfile|
|
||||
folder.first.extract(tempfile.path) { true }
|
||||
image_size = ImageSpec.new(tempfile.path)
|
||||
return [image_size.width, image_size.height]
|
||||
end
|
||||
end
|
||||
|
||||
def self.calculate_dimensions(upload, file)
|
||||
if upload.is_video?
|
||||
video = FFMPEG::Movie.new(file.path)
|
||||
yield(video.width, video.height)
|
||||
|
||||
elsif upload.is_ugoira?
|
||||
w, h = calculate_ugoira_dimensions(file.path)
|
||||
yield(w, h)
|
||||
|
||||
else
|
||||
image_size = ImageSpec.new(file.path)
|
||||
yield(image_size.width, image_size.height)
|
||||
end
|
||||
end
|
||||
|
||||
def self.distribute_files(file, record, type)
|
||||
[Danbooru.config.storage_manager, Danbooru.config.backup_storage_manager].each do |sm|
|
||||
sm.store_file(file, record, type)
|
||||
end
|
||||
end
|
||||
|
||||
def self.is_downloadable?(source)
|
||||
source.match?(/^https?:\/\//)
|
||||
end
|
||||
|
||||
def self.generate_resizes(file, upload)
|
||||
if upload.is_video?
|
||||
video = FFMPEG::Movie.new(file.path)
|
||||
preview_file = generate_video_preview_for(video, Danbooru.config.small_image_width, Danbooru.config.small_image_width)
|
||||
|
||||
elsif upload.is_ugoira?
|
||||
preview_file = PixivUgoiraConverter.generate_preview(file)
|
||||
sample_file = PixivUgoiraConverter.generate_webm(file, upload.context["ugoira"]["frame_data"])
|
||||
|
||||
elsif upload.is_image?
|
||||
preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
|
||||
|
||||
if upload.image_width > Danbooru.config.large_image_width
|
||||
sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, upload.image_height, 90)
|
||||
end
|
||||
end
|
||||
|
||||
[preview_file, sample_file]
|
||||
end
|
||||
|
||||
def self.generate_video_preview_for(video, width, height)
|
||||
dimension_ratio = video.width.to_f / video.height
|
||||
if dimension_ratio > 1
|
||||
height = (width / dimension_ratio).to_i
|
||||
else
|
||||
width = (height * dimension_ratio).to_i
|
||||
end
|
||||
|
||||
output_file = Tempfile.new(binmode: true)
|
||||
video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
|
||||
output_file
|
||||
end
|
||||
|
||||
def self.process_file(upload, file)
|
||||
upload.file = file
|
||||
upload.file_ext = Utils.file_header_to_file_ext(file)
|
||||
upload.file_size = file.size
|
||||
upload.md5 = Digest::MD5.file(file.path).hexdigest
|
||||
|
||||
Utils.calculate_dimensions(upload, file) do |width, height|
|
||||
upload.image_width = width
|
||||
upload.image_height = height
|
||||
end
|
||||
|
||||
upload.tag_string = "#{upload.tag_string} #{Utils.automatic_tags(upload, file)}"
|
||||
|
||||
preview_file, sample_file = Utils.generate_resizes(file, upload)
|
||||
|
||||
begin
|
||||
Utils.distribute_files(file, upload, :original)
|
||||
Utils.distribute_files(sample_file, upload, :large) if sample_file.present?
|
||||
Utils.distribute_files(preview_file, upload, :preview) if preview_file.present?
|
||||
ensure
|
||||
preview_file.try(:close!)
|
||||
sample_file.try(:close!)
|
||||
end
|
||||
end
|
||||
|
||||
# these methods are only really used during upload processing even
|
||||
# though logically they belong on upload. post can rely on the
|
||||
# automatic tag that's added.
|
||||
def self.is_animated_gif?(upload, file)
|
||||
return false if upload.file_ext != "gif"
|
||||
|
||||
# Check whether the gif has multiple frames by trying to load the second frame.
|
||||
result = Vips::Image.gifload(file.path, page: 1) rescue $ERROR_INFO
|
||||
if result.is_a?(Vips::Image)
|
||||
true
|
||||
elsif result.is_a?(Vips::Error) && result.message =~ /too few frames in GIF file/
|
||||
false
|
||||
else
|
||||
raise result
|
||||
end
|
||||
end
|
||||
|
||||
def self.is_animated_png?(upload, file)
|
||||
upload.file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
|
||||
end
|
||||
|
||||
def self.is_video_with_audio?(upload, file)
|
||||
video = FFMPEG::Movie.new(file.path)
|
||||
upload.is_video? && video.audio_channels.present?
|
||||
end
|
||||
|
||||
def self.automatic_tags(upload, file)
|
||||
return "" unless Danbooru.config.enable_dimension_autotagging
|
||||
|
||||
tags = []
|
||||
tags << "video_with_sound" if is_video_with_audio?(upload, file)
|
||||
tags << "animated_gif" if is_animated_gif?(upload, file)
|
||||
tags << "animated_png" if is_animated_png?(upload, file)
|
||||
tags.join(" ")
|
||||
end
|
||||
end
|
||||
|
||||
class Preprocessor
|
||||
attr_reader :params
|
||||
|
||||
def initialize(params)
|
||||
@params = params
|
||||
end
|
||||
|
||||
def source
|
||||
params[:source]
|
||||
end
|
||||
|
||||
def in_progress?
|
||||
Upload.where(status: "preprocessing", source: source).exists?
|
||||
end
|
||||
|
||||
def predecessor
|
||||
Upload.where(status: ["preprocessed", "preprocessing"], source: source).first
|
||||
end
|
||||
|
||||
def completed?
|
||||
predecessor.present?
|
||||
end
|
||||
|
||||
def start!(uploader_id)
|
||||
if !Utils.is_downloadable?(source)
|
||||
return
|
||||
end
|
||||
|
||||
if Post.where(source: source).exists?
|
||||
return
|
||||
end
|
||||
|
||||
if Upload.where(source: source, status: "completed").exists?
|
||||
return
|
||||
end
|
||||
|
||||
if Upload.where(source: source).where("status like ?", "error%").exists?
|
||||
return
|
||||
end
|
||||
|
||||
params[:rating] ||= "q"
|
||||
params[:tag_string] ||= "tagme"
|
||||
|
||||
CurrentUser.as(User.find(uploader_id)) do
|
||||
upload = Upload.create!(params)
|
||||
|
||||
upload.update(status: "preprocessing")
|
||||
|
||||
begin
|
||||
file = download_from_source(source, referer_url: upload.referer_url) do |context|
|
||||
upload.downloaded_source = context[:downloaded_source]
|
||||
upload.source = context[:source]
|
||||
|
||||
if context[:ugoira]
|
||||
upload.context = { ugoira: context[:ugoira] }
|
||||
end
|
||||
end
|
||||
|
||||
Utils.process_file(upload, file)
|
||||
|
||||
upload.rating = params[:rating]
|
||||
upload.tag_string = params[:tag_string]
|
||||
upload.status = "preprocessed"
|
||||
upload.save!
|
||||
rescue Exception => x
|
||||
upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
|
||||
end
|
||||
|
||||
return upload
|
||||
end
|
||||
end
|
||||
|
||||
def finish!
|
||||
pred = self.predecessor()
|
||||
pred.attributes = self.params
|
||||
pred.status = "completed"
|
||||
pred.save
|
||||
return pred
|
||||
end
|
||||
|
||||
def download_from_source(source, referer_url: nil)
|
||||
download = Downloads::File.new(source, referer_url: referer_url)
|
||||
file = download.download!
|
||||
context = {
|
||||
downloaded_source: download.downloaded_source,
|
||||
source: download.source
|
||||
}
|
||||
|
||||
if download.data[:is_ugoira]
|
||||
context[:ugoira] = {
|
||||
frame_data: download.data[:ugoira_frame_data],
|
||||
content_type: download.data[:ugoira_content_type]
|
||||
}
|
||||
end
|
||||
|
||||
yield(context)
|
||||
|
||||
return file
|
||||
end
|
||||
end
|
||||
|
||||
attr_reader :params, :post, :upload
|
||||
|
||||
def initialize(params)
|
||||
@params = params
|
||||
end
|
||||
|
||||
def start!
|
||||
preprocessor = Preprocessor.new(params)
|
||||
|
||||
if preprocessor.in_progress?
|
||||
delay(queue: "default", run_at: 5.seconds.from_now).start!
|
||||
return preprocessor.predecessor
|
||||
end
|
||||
|
||||
if preprocessor.completed?
|
||||
@upload = preprocessor.finish!
|
||||
create_post_from_upload(@upload)
|
||||
return @upload
|
||||
end
|
||||
|
||||
params[:rating] ||= "q"
|
||||
params[:tag_string] ||= "tagme"
|
||||
@upload = Upload.create!(params)
|
||||
|
||||
begin
|
||||
if @upload.invalid?
|
||||
return @upload
|
||||
end
|
||||
|
||||
@upload.update(status: "processing")
|
||||
|
||||
if @upload.file.present?
|
||||
Utils.process_file(upload, @upload.file)
|
||||
else
|
||||
# sources will be handled in preprocessing now
|
||||
end
|
||||
|
||||
@upload.save!
|
||||
@post = create_post_from_upload(@upload)
|
||||
return @upload
|
||||
|
||||
rescue Exception => x
|
||||
@upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
|
||||
@upload
|
||||
end
|
||||
end
|
||||
|
||||
def warnings
|
||||
return [] if @post.nil?
|
||||
return @post.warnings.full_messages
|
||||
end
|
||||
|
||||
def source
|
||||
params[:source]
|
||||
end
|
||||
|
||||
def include_artist_commentary?
|
||||
params[:include_artist_commentary].to_s.truthy?
|
||||
end
|
||||
|
||||
def create_post_from_upload(upload)
|
||||
@post = convert_to_post(upload)
|
||||
@post.save!
|
||||
|
||||
@upload.update(status: "error: " + @post.errors.full_messages.join(", "))
|
||||
|
||||
if upload.context && upload.context["ugoira"]
|
||||
PixivUgoiraFrameData.create(
|
||||
post_id: @post.id,
|
||||
data: upload.context["ugoira"]["frame_data"],
|
||||
content_type: upload.context["ugoira"]["content_type"]
|
||||
)
|
||||
end
|
||||
|
||||
if include_artist_commentary?
|
||||
@post.create_artist_commentary(
|
||||
:original_title => params[:artist_commentary_title],
|
||||
:original_description => params[:artist_commentary_desc]
|
||||
)
|
||||
end
|
||||
|
||||
notify_cropper(@post) if ImageCropper.enabled?
|
||||
upload.update(status: "completed", post_id: @post.id)
|
||||
@post
|
||||
end
|
||||
|
||||
def convert_to_post(upload)
|
||||
Post.new.tap do |p|
|
||||
p.tag_string = upload.tag_string
|
||||
p.md5 = upload.md5
|
||||
p.file_ext = upload.file_ext
|
||||
p.image_width = upload.image_width
|
||||
p.image_height = upload.image_height
|
||||
p.rating = upload.rating
|
||||
p.source = upload.source
|
||||
p.file_size = upload.file_size
|
||||
p.uploader_id = upload.uploader_id
|
||||
p.uploader_ip_addr = upload.uploader_ip_addr
|
||||
p.parent_id = upload.parent_id
|
||||
|
||||
if !upload.uploader.can_upload_free? || upload.upload_as_pending?
|
||||
p.is_pending = true
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def notify_cropper(post)
|
||||
# ImageCropper.notify(post)
|
||||
end
|
||||
end
|
||||
@@ -3,18 +3,67 @@ require "tmpdir"
|
||||
class Upload < ApplicationRecord
|
||||
class Error < Exception ; end
|
||||
|
||||
attr_accessor :file, :image_width, :image_height, :file_ext, :md5,
|
||||
:file_size, :as_pending, :artist_commentary_title,
|
||||
:artist_commentary_desc, :include_artist_commentary,
|
||||
:referer_url, :downloaded_source, :replaced_post
|
||||
belongs_to :uploader, :class_name => "User"
|
||||
class Validator < ActiveModel::Validator
|
||||
def validate(record)
|
||||
if record.new_record?
|
||||
validate_md5_uniqueness(record)
|
||||
validate_video_duration(record)
|
||||
end
|
||||
|
||||
validate_resolution(record)
|
||||
end
|
||||
|
||||
def validate_md5_uniqueness(record)
|
||||
if record.md5.nil?
|
||||
return
|
||||
end
|
||||
|
||||
md5_post = Post.find_by_md5(record.md5)
|
||||
|
||||
if md5_post.nil?
|
||||
return
|
||||
end
|
||||
|
||||
if record.replaced_post && record.replaced_post == md5_post
|
||||
return
|
||||
end
|
||||
|
||||
record.errors[:md5] << "duplicate: #{md5_post.id}"
|
||||
end
|
||||
|
||||
def validate_resolution(record)
|
||||
resolution = record.image_width.to_i * record.image_height.to_i
|
||||
|
||||
if resolution > Danbooru.config.max_image_resolution
|
||||
record.errors[:base] << "image resolution is too large (resolution: #{(resolution / 1_000_000.0).round(1)} megapixels (#{record.image_width}x#{record.image_height}); max: #{Danbooru.config.max_image_resolution / 1_000_000} megapixels)"
|
||||
end
|
||||
end
|
||||
|
||||
def validate_video_duration(record)
|
||||
if record.is_video? && record.video.duration > 120
|
||||
record.errors[:base] << "video must not be longer than 2 minutes"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
attr_accessor :as_pending,
|
||||
:referer_url, :downloaded_source, :replaced_post, :file
|
||||
belongs_to :uploader, :class_name => "User"
|
||||
belongs_to :post, optional: true
|
||||
|
||||
before_validation :initialize_attributes
|
||||
validate :uploader_is_not_limited, :on => :create
|
||||
validate :file_or_source_is_present, :on => :create
|
||||
validate :rating_given
|
||||
before_validation :assign_rating_from_tags
|
||||
validate :uploader_is_not_limited, on: :create
|
||||
# validates :source, format: { with: /\Ahttps?/ }, if: ->(record) {record.file.blank?}, on: :create
|
||||
validates :image_height, numericality: { less_than_or_equal_to: Danbooru.config.max_image_height }, allow_nil: true
|
||||
validates :image_width, numericality: { less_than_or_equal_to: Danbooru.config.max_image_width }, allow_nil: true
|
||||
validates :rating, inclusion: { in: %w(q e s) }, allow_nil: true
|
||||
validates :md5, confirmation: true
|
||||
validates :file_ext, format: { with: /jpg|gif|png|swf|webm|mp4|zip/ }, allow_nil: true
|
||||
validates_with Validator
|
||||
serialize :context, JSON
|
||||
after_create {|rec| rec.uploader.increment!(:post_upload_count)}
|
||||
|
||||
def initialize_attributes
|
||||
self.uploader_id = CurrentUser.user.id
|
||||
@@ -22,189 +71,6 @@ class Upload < ApplicationRecord
|
||||
self.server = Danbooru.config.server_host
|
||||
end
|
||||
|
||||
module ValidationMethods
|
||||
def uploader_is_not_limited
|
||||
if !uploader.can_upload?
|
||||
self.errors.add(:uploader, uploader.upload_limited_reason)
|
||||
return false
|
||||
else
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
def file_or_source_is_present
|
||||
if file.blank? && source.blank?
|
||||
self.errors.add(:base, "Must choose file or specify source")
|
||||
return false
|
||||
else
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
# Because uploads are processed serially, there's no race condition here.
|
||||
def validate_md5_uniqueness
|
||||
md5_post = Post.find_by_md5(md5)
|
||||
|
||||
if md5_post && replaced_post
|
||||
raise "duplicate: #{md5_post.id}" if replaced_post != md5_post
|
||||
elsif md5_post
|
||||
raise "duplicate: #{md5_post.id}"
|
||||
end
|
||||
end
|
||||
|
||||
def validate_file_content_type
|
||||
unless is_valid_content_type?
|
||||
raise "invalid content type (only JPEG, PNG, GIF, SWF, MP4, and WebM files are allowed)"
|
||||
end
|
||||
|
||||
if is_ugoira? && ugoira_service.empty?
|
||||
raise "missing frame data for ugoira"
|
||||
end
|
||||
end
|
||||
|
||||
def validate_md5_confirmation
|
||||
if !md5_confirmation.blank? && md5_confirmation != md5
|
||||
raise "md5 mismatch"
|
||||
end
|
||||
end
|
||||
|
||||
def validate_dimensions
|
||||
resolution = image_width * image_height
|
||||
|
||||
if resolution > Danbooru.config.max_image_resolution
|
||||
raise "image resolution is too large (resolution: #{(resolution / 1_000_000.0).round(1)} megapixels (#{image_width}x#{image_height}); max: #{Danbooru.config.max_image_resolution / 1_000_000} megapixels)"
|
||||
elsif image_width > Danbooru.config.max_image_width
|
||||
raise "image width is too large (width: #{image_width}; max width: #{Danbooru.config.max_image_width})"
|
||||
elsif image_height > Danbooru.config.max_image_height
|
||||
raise "image height is too large (height: #{image_height}; max height: #{Danbooru.config.max_image_height})"
|
||||
end
|
||||
end
|
||||
|
||||
def rating_given
|
||||
if rating.present?
|
||||
return true
|
||||
elsif tag_string =~ /(?:\s|^)rating:([qse])/i
|
||||
self.rating = $1.downcase
|
||||
return true
|
||||
else
|
||||
self.errors.add(:base, "Must specify a rating")
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
def automatic_tags
|
||||
return "" unless Danbooru.config.enable_dimension_autotagging
|
||||
|
||||
tags = []
|
||||
tags << "video_with_sound" if is_video_with_audio?
|
||||
tags << "animated_gif" if is_animated_gif?
|
||||
tags << "animated_png" if is_animated_png?
|
||||
tags.join(" ")
|
||||
end
|
||||
|
||||
def validate_video_duration
|
||||
unless uploader.is_admin?
|
||||
if is_video? && video.duration > 120
|
||||
raise "video must not be longer than 2 minutes"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module ConversionMethods
|
||||
def process_upload
|
||||
begin
|
||||
update_attribute(:status, "processing")
|
||||
|
||||
self.source = source.to_s.strip
|
||||
if is_downloadable?
|
||||
self.downloaded_source, self.source, self.file = download_from_source(source, referer_url)
|
||||
elsif self.file.respond_to?(:tempfile)
|
||||
self.file = self.file.tempfile
|
||||
end
|
||||
|
||||
self.file_ext = file_header_to_file_ext(file)
|
||||
self.file_size = file.size
|
||||
self.md5 = Digest::MD5.file(file.path).hexdigest
|
||||
|
||||
validate_file_content_type
|
||||
validate_md5_uniqueness
|
||||
validate_md5_confirmation
|
||||
validate_video_duration
|
||||
|
||||
self.tag_string = "#{tag_string} #{automatic_tags}"
|
||||
self.image_width, self.image_height = calculate_dimensions
|
||||
validate_dimensions
|
||||
|
||||
save
|
||||
end
|
||||
end
|
||||
|
||||
def create_post_from_upload
|
||||
post = convert_to_post
|
||||
distribute_files(post)
|
||||
|
||||
if post.save
|
||||
create_artist_commentary(post) if include_artist_commentary?
|
||||
ugoira_service.save_frame_data(post) if is_ugoira?
|
||||
notify_cropper(post)
|
||||
update_attributes(:status => "completed", :post_id => post.id)
|
||||
else
|
||||
update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
|
||||
end
|
||||
|
||||
post
|
||||
end
|
||||
|
||||
def distribute_files(post)
|
||||
preview_file, sample_file = generate_resizes
|
||||
post.distribute_files(file, sample_file, preview_file)
|
||||
ensure
|
||||
preview_file.try(:close!)
|
||||
sample_file.try(:close!)
|
||||
end
|
||||
|
||||
def process!(force = false)
|
||||
process_upload
|
||||
post = create_post_from_upload
|
||||
rescue Exception => x
|
||||
update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
|
||||
nil
|
||||
ensure
|
||||
file.try(:close!)
|
||||
end
|
||||
|
||||
def ugoira_service
|
||||
@ugoira_service ||= PixivUgoiraService.new
|
||||
end
|
||||
|
||||
def convert_to_post
|
||||
Post.new.tap do |p|
|
||||
p.tag_string = tag_string
|
||||
p.md5 = md5
|
||||
p.file_ext = file_ext
|
||||
p.image_width = image_width
|
||||
p.image_height = image_height
|
||||
p.rating = rating
|
||||
p.source = source
|
||||
p.file_size = file_size
|
||||
p.uploader_id = uploader_id
|
||||
p.uploader_ip_addr = uploader_ip_addr
|
||||
p.parent_id = parent_id
|
||||
|
||||
if !uploader.can_upload_free? || upload_as_pending?
|
||||
p.is_pending = true
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def notify_cropper(post)
|
||||
if ImageCropper.enabled?
|
||||
# ImageCropper.notify(post)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module FileMethods
|
||||
def is_image?
|
||||
%w(jpg gif png).include?(file_ext)
|
||||
@@ -218,120 +84,9 @@ class Upload < ApplicationRecord
|
||||
%w(webm mp4).include?(file_ext)
|
||||
end
|
||||
|
||||
def is_video_with_audio?
|
||||
is_video? && video.audio_channels.present?
|
||||
end
|
||||
|
||||
def is_ugoira?
|
||||
%w(zip).include?(file_ext)
|
||||
end
|
||||
|
||||
def is_animated_gif?
|
||||
return false if file_ext != "gif"
|
||||
|
||||
# Check whether the gif has multiple frames by trying to load the second frame.
|
||||
result = Vips::Image.gifload(file.path, page: 1) rescue $ERROR_INFO
|
||||
if result.is_a?(Vips::Image)
|
||||
true
|
||||
elsif result.is_a?(Vips::Error) && result.message =~ /too few frames in GIF file/
|
||||
false
|
||||
else
|
||||
raise result
|
||||
end
|
||||
end
|
||||
|
||||
def is_animated_png?
|
||||
file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
|
||||
end
|
||||
end
|
||||
|
||||
module ResizerMethods
|
||||
def generate_resizes
|
||||
if is_video?
|
||||
preview_file = generate_video_preview_for(video, Danbooru.config.small_image_width, Danbooru.config.small_image_width)
|
||||
elsif is_ugoira?
|
||||
preview_file = PixivUgoiraConverter.generate_preview(file)
|
||||
sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data)
|
||||
elsif is_image?
|
||||
preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
|
||||
|
||||
if image_width > Danbooru.config.large_image_width
|
||||
sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, image_height, 90)
|
||||
end
|
||||
end
|
||||
|
||||
[preview_file, sample_file]
|
||||
end
|
||||
|
||||
def generate_video_preview_for(video, width, height)
|
||||
dimension_ratio = video.width.to_f / video.height
|
||||
if dimension_ratio > 1
|
||||
height = (width / dimension_ratio).to_i
|
||||
else
|
||||
width = (height * dimension_ratio).to_i
|
||||
end
|
||||
|
||||
output_file = Tempfile.new(binmode: true)
|
||||
video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
|
||||
output_file
|
||||
end
|
||||
end
|
||||
|
||||
module DimensionMethods
|
||||
# Figures out the dimensions of the image.
|
||||
def calculate_dimensions
|
||||
if is_video?
|
||||
[video.width, video.height]
|
||||
elsif is_ugoira?
|
||||
ugoira_service.calculate_dimensions(file.path)
|
||||
[ugoira_service.width, ugoira_service.height]
|
||||
else
|
||||
image_size = ImageSpec.new(file.path)
|
||||
[image_size.width, image_size.height]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module ContentTypeMethods
|
||||
def is_valid_content_type?
|
||||
file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/
|
||||
end
|
||||
|
||||
def file_header_to_file_ext(file)
|
||||
case File.read(file.path, 16)
|
||||
when /^\xff\xd8/n
|
||||
"jpg"
|
||||
when /^GIF87a/, /^GIF89a/
|
||||
"gif"
|
||||
when /^\x89PNG\r\n\x1a\n/n
|
||||
"png"
|
||||
when /^CWS/, /^FWS/, /^ZWS/
|
||||
"swf"
|
||||
when /^\x1a\x45\xdf\xa3/n
|
||||
"webm"
|
||||
when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
|
||||
"mp4"
|
||||
when /^PK\x03\x04/
|
||||
"zip"
|
||||
else
|
||||
"bin"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module DownloaderMethods
|
||||
# Determines whether the source is downloadable
|
||||
def is_downloadable?
|
||||
source =~ /^https?:\/\// && file.blank?
|
||||
end
|
||||
|
||||
def download_from_source(source, referer_url = nil)
|
||||
download = Downloads::File.new(source, referer_url: referer_url)
|
||||
file = download.download!
|
||||
ugoira_service.load(download.data)
|
||||
|
||||
[download.downloaded_source, download.source, file]
|
||||
end
|
||||
end
|
||||
|
||||
module StatusMethods
|
||||
@@ -347,6 +102,14 @@ class Upload < ApplicationRecord
|
||||
status == "completed"
|
||||
end
|
||||
|
||||
def is_preprocessed?
|
||||
status == "preprocessed"
|
||||
end
|
||||
|
||||
def is_preprocessing?
|
||||
status == "preprocessing"
|
||||
end
|
||||
|
||||
def is_duplicate?
|
||||
status =~ /duplicate/
|
||||
end
|
||||
@@ -448,28 +211,27 @@ class Upload < ApplicationRecord
|
||||
end
|
||||
end
|
||||
|
||||
module ArtistCommentaryMethods
|
||||
def create_artist_commentary(post)
|
||||
post.create_artist_commentary(
|
||||
:original_title => artist_commentary_title,
|
||||
:original_description => artist_commentary_desc
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
include ConversionMethods
|
||||
include ValidationMethods
|
||||
include FileMethods
|
||||
include ResizerMethods
|
||||
include DimensionMethods
|
||||
include ContentTypeMethods
|
||||
include DownloaderMethods
|
||||
include StatusMethods
|
||||
include UploaderMethods
|
||||
include VideoMethods
|
||||
extend SearchMethods
|
||||
include ApiMethods
|
||||
include ArtistCommentaryMethods
|
||||
|
||||
def uploader_is_not_limited
|
||||
if !uploader.can_upload?
|
||||
self.errors.add(:uploader, uploader.upload_limited_reason)
|
||||
return false
|
||||
else
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
def assign_rating_from_tags
|
||||
if tag_string =~ /(?:\s|^)rating:([qse])/i
|
||||
self.rating = $1.downcase
|
||||
end
|
||||
end
|
||||
|
||||
def presenter
|
||||
@presenter ||= UploadPresenter.new(self)
|
||||
@@ -478,8 +240,4 @@ class Upload < ApplicationRecord
|
||||
def upload_as_pending?
|
||||
as_pending.to_s.truthy?
|
||||
end
|
||||
|
||||
def include_artist_commentary?
|
||||
include_artist_commentary.to_s.truthy?
|
||||
end
|
||||
end
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<p>This upload has finished processing. <%= link_to "View the post", post_path(@upload.post_id) %>.</p>
|
||||
<% elsif @upload.is_pending? %>
|
||||
<p>This upload is waiting to be processed. Please wait a few seconds.</p>
|
||||
<% elsif @upload.is_processing? %>
|
||||
<% elsif @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
|
||||
<p>This upload is being processed. Please wait a few seconds.</p>
|
||||
<% elsif @upload.is_duplicate? %>
|
||||
<p>This upload is a duplicate: <%= link_to "post ##{@upload.duplicate_post_id}", post_path(@upload.duplicate_post_id) %></p>
|
||||
@@ -42,7 +42,7 @@
|
||||
Upload - <%= Danbooru.config.app_name %>
|
||||
<% end %>
|
||||
|
||||
<% if @upload.is_pending? || @upload.is_processing? %>
|
||||
<% if @upload.is_pending? || @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
|
||||
<% content_for(:html_header) do %>
|
||||
<meta http-equiv="refresh" content="2">
|
||||
<% end %>
|
||||
|
||||
Reference in New Issue
Block a user