jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #3718 from r888888888/async-uploads

Browse Source 
Asynchronous Uploads
This commit is contained in:
Albert Yi
2018-06-15 14:53:37 -07:00
committed by GitHub
parent b561a6d9ab 06f47cc84d
commit c89c861326
25 changed files with 1803 additions and 1117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Procfile
View File

@@ -1,2 +1,2 @@
unicorn: bundle exec rails server
jobs: bundle exec script/delayed_job run
jobs: bundle exec script/delayed_job run

4
app/assets/javascripts/related_tag.js.erb
View File

@@ -34,7 +34,6 @@
$dest.empty();
Danbooru.RelatedTag.build_recent_and_frequent($dest);
$dest.append("<em>Loading...</em>");
$("#related-tags-container").show();
$.get("/related_tag.json", {
"query": Danbooru.RelatedTag.current_tag(),
"category": category
@@ -93,6 +92,9 @@
}
Danbooru.RelatedTag.process_response = function(data) {
if (data.tags.length || data.wiki_page_tags.length || data.other_wikis.length) {
$("#related-tags-container").show();
}
Danbooru.RelatedTag.recent_search = data;
Danbooru.RelatedTag.build_all();
}

2
app/assets/javascripts/uploads.js
View File

@@ -33,7 +33,7 @@
Danbooru.Upload.initialize_submit = function() {
$("#form").submit(function(e) {
var error_messages = [];
if (($("#upload_file").val() === "") && ($("#upload_source").val() === "")) {
if (($("#upload_file").val() === "") && ($("#upload_source").val() === "") && $("#upload_md5_confirmation").val() === "") {
error_messages.push("Must choose file or specify source");
}
if (!$("#upload_rating_s").prop("checked") && !$("#upload_rating_q").prop("checked") && !$("#upload_rating_e").prop("checked") &&

3
app/assets/stylesheets/specific/posts.scss
View File

@@ -471,9 +471,10 @@ div#c-post-versions, div#c-artist-versions {
div#c-posts, div#c-uploads {
/* Fetch source data box */
div#source-info {
border-radius: 4px;
margin: 1em 0;
padding: 1em;
border: 1px solid gray;
border: 1px solid #666;
p {
margin: 0;

2
app/assets/stylesheets/specific/related_tags.scss
View File

@@ -1,6 +1,7 @@
@import "../common/000_vars.scss";
div#related-tags-container {
display: none;
padding-right: 2em;
h1 {
@@ -14,6 +15,7 @@ div.related-tags {
padding: 1em;
background: #EEE;
overflow: auto;
border-radius: 4px;
div.tag-column {
max-width: 15em;

42
app/assets/stylesheets/specific/uploads.scss
View File

@@ -34,6 +34,48 @@ div#c-uploads {
div.field_with_errors {
display: inline;
}
#filedropzone {
border: 4px dashed #DDD;
padding: 0;
min-height: 100px;
display: flex;
align-items: center;
justify-content: center;
border-radius: 4px;
.placeholder {
font-style: italic;
color: #333;
height: 100%;
}
&.error {
border-color: darken(#f2dede, 30%);
background-color: #f2dede;
}
&.success {
border-color: darken(#dff0d8, 30%);
background-color: #dff0d8;
}
}
.dz-preview {
margin-bottom: 1em;
}
.dz-progress {
height: 20px;
width: 300px;
border: 1px solid #CCC;
.dz-upload {
background-color: #F5F5FF;
display: block;
height: 20px;
}
}
}
div#a-index {

52
app/controllers/uploads_controller.rb
View File

@@ -1,33 +1,19 @@
class UploadsController < ApplicationController
before_action :member_only, except: [:index, :show]
respond_to :html, :xml, :json, :js
skip_before_action :verify_authenticity_token, only: [:preprocess]
def new
@upload = Upload.new
@upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
if params[:url]
download = Downloads::File.new(params[:url])
@normalized_url, _, _ = download.before_download(params[:url], {})
@post = find_post_by_url(@normalized_url)
begin
@source = Sources::Site.new(params[:url], :referer_url => params[:ref])
@remote_size = download.size
rescue Exception
end
end
@upload, @post, @source, @normalized_url, @remote_size = UploadService::ControllerHelper.prepare(
url: params[:url], ref: params[:ref]
)
respond_with(@upload)
end
def batch
@url = params.dig(:batch, :url) || params[:url]
@source = nil
if @url
@source = Sources::Site.new(@url, :referer_url => params[:ref])
@source.get
end
@source = UploadService::ControllerHelper.batch(@url, params[:ref])
respond_with(@source)
end
@@ -56,15 +42,19 @@ class UploadsController < ApplicationController
end
end
def preprocess
@upload, @post, @source, @normalized_url, @remote_size = UploadService::ControllerHelper.prepare(
url: params[:url], file: params[:file], ref: params[:ref]
)
render body: nil
end
def create
@upload = Upload.create(upload_params)
@service = UploadService.new(upload_params)
@upload = @service.start!
if @upload.errors.empty?
post = @upload.process!
if post.present? && post.valid? && post.warnings.any?
flash[:notice] = post.warnings.full_messages.join(".\n \n")
end
if @service.warnings.any?
flash[:notice] = @service.warnings.join(".\n \n")
end
save_recent_tags
@@ -73,14 +63,6 @@ class UploadsController < ApplicationController
private
def find_post_by_url(normalized_url)
if normalized_url.nil?
Post.where("SourcePattern(lower(posts.source)) = ?", params[:url]).first
else
Post.where("SourcePattern(lower(posts.source)) IN (?)", [params[:url], @normalized_url]).first
end
end
def save_recent_tags
if @upload
tags = Tag.scan_tags(@upload.tag_string)
@@ -94,7 +76,7 @@ class UploadsController < ApplicationController
permitted_params = %i[
file source tag_string rating status parent_id artist_commentary_title
artist_commentary_desc include_artist_commentary referer_url
md5_confirmation as_pending
md5_confirmation as_pending
]
params.require(:upload).permit(permitted_params)

33
app/logical/pixiv_ugoira_service.rb
View File

@@ -1,33 +0,0 @@
class PixivUgoiraService
attr_reader :width, :height, :frame_data, :content_type
def save_frame_data(post)
PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id)
end
def calculate_dimensions(source_path)
folder = Zip::File.new(source_path)
tempfile = Tempfile.new("ugoira-dimensions")
begin
folder.first.extract(tempfile.path) {true}
image_size = ImageSpec.new(tempfile.path)
@width = image_size.width
@height = image_size.height
ensure
tempfile.close
tempfile.unlink
end
end
def load(data)
if data[:is_ugoira]
@frame_data = data[:ugoira_frame_data]
@content_type = data[:ugoira_content_type]
end
end
def empty?
@frame_data.nil?
end
end

587
app/logical/upload_service.rb Normal file
View File

@@ -0,0 +1,587 @@
class UploadService
module ControllerHelper
def self.prepare(url: nil, file: nil, ref: nil)
upload = Upload.new
if url
# this gets called from UploadsController#new so we need
# to preprocess async
Preprocessor.new(source: url).delay(queue: "default").start!(CurrentUser.user.id)
download = Downloads::File.new(url)
normalized_url, _, _ = download.before_download(url, {})
post = if normalized_url.nil?
Post.where("SourcePattern(lower(posts.source)) = ?", url).first
else
Post.where("SourcePattern(lower(posts.source)) IN (?)", [url, normalized_url]).first
end
begin
source = Sources::Site.new(url, :referer_url => ref)
remote_size = download.size
rescue Exception
end
return [upload, post, source, normalized_url, remote_size]
elsif file
# this gets called via XHR so we can process sync
Preprocessor.new(file: file).start!((CurrentUser.user.id))
end
return [upload]
end
def self.batch(url, ref = nil)
if url
source = Sources::Site.new(url, :referer_url => ref)
source.get
return source
end
end
end
module Utils
def self.file_header_to_file_ext(file)
case File.read(file.path, 16)
when /^\xff\xd8/n
"jpg"
when /^GIF87a/, /^GIF89a/
"gif"
when /^\x89PNG\r\n\x1a\n/n
"png"
when /^CWS/, /^FWS/, /^ZWS/
"swf"
when /^\x1a\x45\xdf\xa3/n
"webm"
when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
"mp4"
when /^PK\x03\x04/
"zip"
else
"bin"
end
end
def self.delete_file(md5, file_ext)
if Post.where(md5: md5).exists?
return
end
Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :original)
Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :large)
Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :preview)
Danbooru.config.backup_storage_manager.delete_file(nil, md5, file_ext, :original)
Danbooru.config.backup_storage_manager.delete_file(nil, md5, file_ext, :large)
Danbooru.config.backup_storage_manager.delete_file(nil, md5, file_ext, :preview)
end
def self.calculate_ugoira_dimensions(source_path)
folder = Zip::File.new(source_path)
Tempfile.open("ugoira-dim-") do |tempfile|
folder.first.extract(tempfile.path) { true }
image_size = ImageSpec.new(tempfile.path)
return [image_size.width, image_size.height]
end
end
def self.calculate_dimensions(upload, file)
if upload.is_video?
video = FFMPEG::Movie.new(file.path)
yield(video.width, video.height)
elsif upload.is_ugoira?
w, h = calculate_ugoira_dimensions(file.path)
yield(w, h)
else
image_size = ImageSpec.new(file.path)
yield(image_size.width, image_size.height)
end
end
def self.distribute_files(file, record, type)
[Danbooru.config.storage_manager, Danbooru.config.backup_storage_manager].each do |sm|
sm.store_file(file, record, type)
end
end
def self.is_downloadable?(source)
source.match?(/^https?:\/\//)
end
def self.generate_resizes(file, upload)
if upload.is_video?
video = FFMPEG::Movie.new(file.path)
preview_file = generate_video_preview_for(video, Danbooru.config.small_image_width, Danbooru.config.small_image_width)
elsif upload.is_ugoira?
preview_file = PixivUgoiraConverter.generate_preview(file)
sample_file = PixivUgoiraConverter.generate_webm(file, upload.context["ugoira"]["frame_data"])
elsif upload.is_image?
preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
if upload.image_width > Danbooru.config.large_image_width
sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, upload.image_height, 90)
end
end
[preview_file, sample_file]
end
def self.generate_video_preview_for(video, width, height)
dimension_ratio = video.width.to_f / video.height
if dimension_ratio > 1
height = (width / dimension_ratio).to_i
else
width = (height * dimension_ratio).to_i
end
output_file = Tempfile.new(binmode: true)
video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
output_file
end
def self.process_file(upload, file)
upload.file = file
upload.file_ext = Utils.file_header_to_file_ext(file)
upload.file_size = file.size
upload.md5 = Digest::MD5.file(file.path).hexdigest
if Post.where(md5: upload.md5).exists?
# abort early if this post already exists
raise Upload::Error.new("Post with MD5 #{upload.md5} already exists")
end
Utils.calculate_dimensions(upload, file) do |width, height|
upload.image_width = width
upload.image_height = height
end
upload.tag_string = "#{upload.tag_string} #{Utils.automatic_tags(upload, file)}"
preview_file, sample_file = Utils.generate_resizes(file, upload)
begin
Utils.distribute_files(file, upload, :original)
Utils.distribute_files(sample_file, upload, :large) if sample_file.present?
Utils.distribute_files(preview_file, upload, :preview) if preview_file.present?
ensure
preview_file.try(:close!)
sample_file.try(:close!)
end
# in case this upload never finishes processing, we need to delete the
# distributed files in the future
Danbooru.config.other_server_hosts.each do |host|
UploadService::Utils.delay(queue: host, run_at: 10.minutes.from_now).delete_file(upload.md5, upload.file_ext)
end
end
# these methods are only really used during upload processing even
# though logically they belong on upload. post can rely on the
# automatic tag that's added.
def self.is_animated_gif?(upload, file)
return false if upload.file_ext != "gif"
# Check whether the gif has multiple frames by trying to load the second frame.
result = Vips::Image.gifload(file.path, page: 1) rescue $ERROR_INFO
if result.is_a?(Vips::Image)
true
elsif result.is_a?(Vips::Error) && result.message =~ /too few frames in GIF file/
false
else
raise result
end
end
def self.is_animated_png?(upload, file)
upload.file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
end
def self.is_video_with_audio?(upload, file)
video = FFMPEG::Movie.new(file.path)
upload.is_video? && video.audio_channels.present?
end
def self.automatic_tags(upload, file)
return "" unless Danbooru.config.enable_dimension_autotagging
tags = []
tags << "video_with_sound" if is_video_with_audio?(upload, file)
tags << "animated_gif" if is_animated_gif?(upload, file)
tags << "animated_png" if is_animated_png?(upload, file)
tags.join(" ")
end
end
class Preprocessor
attr_reader :params
def initialize(params)
@params = params
end
def source
params[:source]
end
def md5
params[:md5_confirmation]
end
def in_progress?
if source.present?
Upload.where(status: "preprocessing", source: source).exists?
elsif md5.present?
Upload.where(status: "preprocessing", md5: md5).exists?
else
false
end
end
def predecessor
if source.present?
Upload.where(status: ["preprocessed", "preprocessing"], source: source).first
elsif md5.present?
Upload.where(status: ["preprocessed", "preprocessing"], md5: md5).first
end
end
def completed?
predecessor.present?
end
def start!(uploader_id)
if source.present?
if !Utils.is_downloadable?(source)
return
end
if Post.where(source: source).exists?
return
end
if Upload.where(source: source, status: "completed").exists?
return
end
if Upload.where(source: source).where("status like ?", "error%").exists?
return
end
end
params[:rating] ||= "q"
params[:tag_string] ||= "tagme"
CurrentUser.as(User.find(uploader_id)) do
upload = Upload.create!(params)
upload.update(status: "preprocessing")
begin
if source.present?
file = download_from_source(source, referer_url: upload.referer_url) do |context|
upload.downloaded_source = context[:downloaded_source]
upload.source = context[:source]
if context[:ugoira]
upload.context = { ugoira: context[:ugoira] }
end
end
elsif params[:file].present?
file = params[:file]
end
Utils.process_file(upload, file)
upload.rating = params[:rating]
upload.tag_string = params[:tag_string]
upload.status = "preprocessed"
upload.save!
rescue Exception => x
upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
end
return upload
end
end
def finish!(upload = nil)
pred = upload || self.predecessor()
pred.attributes = self.params
pred.status = "completed"
pred.save
return pred
end
def download_from_source(source, referer_url: nil)
download = Downloads::File.new(source, referer_url: referer_url)
file = download.download!
context = {
downloaded_source: download.downloaded_source,
source: download.source
}
if download.data[:is_ugoira]
context[:ugoira] = {
frame_data: download.data[:ugoira_frame_data],
content_type: download.data[:ugoira_content_type]
}
end
yield(context)
return file
end
end
class Replacer
attr_reader :post, :replacement
def initialize(post:, replacement:)
@post = post
@replacement = replacement
end
def comment_replacement_message(post, replacement)
%("#{replacement.creator.name}":[/users/#{replacement.creator.id}] replaced this post with a new image:\n\n#{replacement_message(post, replacement)})
end
def replacement_message(post, replacement)
linked_source = linked_source(replacement.replacement_url)
linked_source_was = linked_source(post.source_was)
<<-EOS.strip_heredoc
[table]
[tbody]
[tr]
[th]Old[/th]
[td]#{linked_source_was}[/td]
[td]#{post.md5_was}[/td]
[td]#{post.file_ext_was}[/td]
[td]#{post.image_width_was} x #{post.image_height_was}[/td]
[td]#{post.file_size_was.to_s(:human_size, precision: 4)}[/td]
[/tr]
[tr]
[th]New[/th]
[td]#{linked_source}[/td]
[td]#{post.md5}[/td]
[td]#{post.file_ext}[/td]
[td]#{post.image_width} x #{post.image_height}[/td]
[td]#{post.file_size.to_s(:human_size, precision: 4)}[/td]
[/tr]
[/tbody]
[/table]
EOS
end
def linked_source(source)
return nil if source.nil?
# truncate long sources in the middle: "www.pixiv.net...lust_id=23264933"
truncated_source = source.gsub(%r{\Ahttps?://}, "").truncate(64, omission: "...#{source.last(32)}")
if source =~ %r{\Ahttps?://}i
%("#{truncated_source}":[#{source}])
else
truncated_source
end
end
def undo!
undo_replacement = post.replacements.create(replacement_url: replacement.original_url)
undoer = Replacer.new(post: post, replacement: undo_replacement)
undoer.process!
end
def process!
preprocessor = Preprocessor.new(
rating: post.rating,
tag_string: replacement.tags,
source: replacement.replacement_url,
file: replacement.replacement_file
)
upload = preprocessor.start!(CurrentUser.id)
return if upload.is_errored?
upload = preprocessor.finish!(upload)
return if upload.is_errored?
md5_changed = upload.md5 != post.md5
if replacement.replacement_file.present?
replacement.replacement_url = "file://#{replacement.replacement_file.original_filename}"
elsif upload.downloaded_source.present?
replacement.replacement_url = upload.downloaded_source
end
if md5_changed
post.queue_delete_files(PostReplacement::DELETION_GRACE_PERIOD)
end
replacement.file_ext = upload.file_ext
replacement.file_size = upload.file_size
replacement.image_height = upload.image_height
replacement.image_width = upload.image_width
replacement.md5 = upload.md5
post.md5 = upload.md5
post.file_ext = upload.file_ext
post.image_width = upload.image_width
post.image_height = upload.image_height
post.file_size = upload.file_size
post.source = upload.downloaded_source || upload.source
post.tag_string = upload.tag_string
update_ugoira_frame_data(post, upload)
if md5_changed
CurrentUser.as(User.system) do
post.comments.create!(body: comment_replacement_message(post, replacement), do_not_bump_post: true)
end
end
if replacement.final_source.present?
post.update(source: replacement.final_source)
end
replacement.save!
post.save!
rescale_notes(post)
end
def rescale_notes(post)
x_scale = post.image_width.to_f / post.image_width_before_last_save.to_f
y_scale = post.image_height.to_f / post.image_height_before_last_save.to_f
post.notes.each do |note|
note.reload
note.rescale!(x_scale, y_scale)
end
end
def update_ugoira_frame_data(post, upload)
post.pixiv_ugoira_frame_data.destroy if post.pixiv_ugoira_frame_data.present?
unless post.is_ugoira?
return
end
PixivUgoiraFrameData.create(
post_id: post.id,
data: upload.context["ugoira"]["frame_data"],
content_type: upload.context["ugoira"]["content_type"]
)
end
end
attr_reader :params, :post, :upload
def initialize(params)
@params = params
end
def start!
preprocessor = Preprocessor.new(params)
if preprocessor.in_progress?
delay(queue: "default", run_at: 5.seconds.from_now).start!
return preprocessor.predecessor
end
if preprocessor.completed?
@upload = preprocessor.finish!
create_post_from_upload(@upload)
return @upload
end
params[:rating] ||= "q"
params[:tag_string] ||= "tagme"
@upload = Upload.create!(params)
begin
if @upload.invalid?
return @upload
end
@upload.update(status: "processing")
if @upload.file.present?
Utils.process_file(upload, @upload.file)
else
# sources will be handled in preprocessing now
end
@upload.save!
@post = create_post_from_upload(@upload)
return @upload
rescue Exception => x
@upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
@upload
end
end
def warnings
return [] if @post.nil?
return @post.warnings.full_messages
end
def source
params[:source]
end
def include_artist_commentary?
params[:include_artist_commentary].to_s.truthy?
end
def create_post_from_upload(upload)
@post = convert_to_post(upload)
@post.save!
upload.update(status: "error: " + @post.errors.full_messages.join(", "))
if upload.context && upload.context["ugoira"]
PixivUgoiraFrameData.create(
post_id: @post.id,
data: upload.context["ugoira"]["frame_data"],
content_type: upload.context["ugoira"]["content_type"]
)
end
if include_artist_commentary?
@post.create_artist_commentary(
:original_title => params[:artist_commentary_title],
:original_description => params[:artist_commentary_desc]
)
end
notify_cropper(@post) if ImageCropper.enabled?
upload.update(status: "completed", post_id: @post.id)
@post
end
def convert_to_post(upload)
Post.new.tap do |p|
p.tag_string = upload.tag_string
p.md5 = upload.md5
p.file_ext = upload.file_ext
p.image_width = upload.image_width
p.image_height = upload.image_height
p.rating = upload.rating
p.source = upload.source
p.file_size = upload.file_size
p.uploader_id = upload.uploader_id
p.uploader_ip_addr = upload.uploader_ip_addr
p.parent_id = upload.parent_id
if !upload.uploader.can_upload_free? || upload.upload_as_pending?
p.is_pending = true
end
end
end
def notify_cropper(post)
# ImageCropper.notify(post)
end
end

3
app/models/post.rb
View File

@@ -1396,7 +1396,8 @@ class Post < ApplicationRecord
def replace!(params)
transaction do
replacement = replacements.create(params)
replacement.process!
processor = UploadService::Replacer.new(post: self, replacement: replacement)
processor.process!
replacement
end
end

178
app/models/post_replacement.rb
View File

@@ -18,166 +18,40 @@ class PostReplacement < ApplicationRecord
self.md5_was = post.md5
end
def undo!
undo_replacement = post.replacements.create(replacement_url: original_url)
undo_replacement.process!
end
def process!
upload = nil
transaction do
upload = Upload.create!(
file: replacement_file,
source: replacement_url,
rating: post.rating,
tag_string: self.tags,
replaced_post: post,
)
upload.process_upload
upload.update(status: "completed", post_id: post.id)
md5_changed = (upload.md5 != post.md5)
if replacement_file.present?
self.replacement_url = "file://#{replacement_file.original_filename}"
else
self.replacement_url = upload.downloaded_source
concerning :Search do
class_methods do
def post_tags_match(query)
PostQueryBuilder.new(query).build(self.joins(:post))
end
# queue the deletion *before* updating the post so that we use the old
# md5/file_ext to delete the old files. if saving the post fails,
# this is rolled back so the job won't run.
if md5_changed
post.queue_delete_files(DELETION_GRACE_PERIOD)
end
def search(params = {})
q = super
self.file_ext = upload.file_ext
self.file_size = upload.file_size
self.image_height = upload.image_height
self.image_width = upload.image_width
self.md5 = upload.md5
post.md5 = upload.md5
post.file_ext = upload.file_ext
post.image_width = upload.image_width
post.image_height = upload.image_height
post.file_size = upload.file_size
post.source = final_source.presence || upload.source
post.tag_string = upload.tag_string
rescale_notes
update_ugoira_frame_data(upload)
if md5_changed
CurrentUser.as(User.system) do
post.comments.create!(body: comment_replacement_message, do_not_bump_post: true)
if params[:creator_id].present?
q = q.where(creator_id: params[:creator_id].split(",").map(&:to_i))
end
if params[:creator_name].present?
q = q.where(creator_id: User.name_to_id(params[:creator_name]))
end
if params[:post_id].present?
q = q.where(post_id: params[:post_id].split(",").map(&:to_i))
end
if params[:post_tags_match].present?
q = q.post_tags_match(params[:post_tags_match])
end
q.apply_default_order(params)
end
save!
post.save!
end
# point of no return: these things can't be rolled back, so we do them
# only after the transaction successfully commits.
upload.distribute_files(post)
post.update_iqdb_async
end
def rescale_notes
x_scale = post.image_width.to_f / post.image_width_was.to_f
y_scale = post.image_height.to_f / post.image_height_was.to_f
post.notes.each do |note|
note.rescale!(x_scale, y_scale)
end
end
def update_ugoira_frame_data(upload)
post.pixiv_ugoira_frame_data.destroy if post.pixiv_ugoira_frame_data.present?
upload.ugoira_service.save_frame_data(post) if post.is_ugoira?
def suggested_tags_for_removal
tags = post.tag_array.select { |tag| Danbooru.config.remove_tag_after_replacement?(tag) }
tags = tags.map { |tag| "-#{tag}" }
tags.join(" ")
end
module SearchMethods
def post_tags_match(query)
PostQueryBuilder.new(query).build(self.joins(:post))
end
def search(params = {})
q = super
if params[:creator_id].present?
q = q.where(creator_id: params[:creator_id].split(",").map(&:to_i))
end
if params[:creator_name].present?
q = q.where(creator_id: User.name_to_id(params[:creator_name]))
end
if params[:post_id].present?
q = q.where(post_id: params[:post_id].split(",").map(&:to_i))
end
if params[:post_tags_match].present?
q = q.post_tags_match(params[:post_tags_match])
end
q.apply_default_order(params)
end
end
module PresenterMethods
def comment_replacement_message
%("#{creator.name}":[/users/#{creator.id}] replaced this post with a new image:\n\n#{replacement_message})
end
def replacement_message
linked_source = linked_source(replacement_url)
linked_source_was = linked_source(post.source_was)
<<-EOS.strip_heredoc
[table]
[tbody]
[tr]
[th]Old[/th]
[td]#{linked_source_was}[/td]
[td]#{post.md5_was}[/td]
[td]#{post.file_ext_was}[/td]
[td]#{post.image_width_was} x #{post.image_height_was}[/td]
[td]#{post.file_size_was.to_s(:human_size, precision: 4)}[/td]
[/tr]
[tr]
[th]New[/th]
[td]#{linked_source}[/td]
[td]#{post.md5}[/td]
[td]#{post.file_ext}[/td]
[td]#{post.image_width} x #{post.image_height}[/td]
[td]#{post.file_size.to_s(:human_size, precision: 4)}[/td]
[/tr]
[/tbody]
[/table]
EOS
end
def linked_source(source)
# truncate long sources in the middle: "www.pixiv.net...lust_id=23264933"
truncated_source = source.gsub(%r{\Ahttps?://}, "").truncate(64, omission: "...#{source.last(32)}")
if source =~ %r{\Ahttps?://}i
%("#{truncated_source}":[#{source}])
else
truncated_source
end
end
def suggested_tags_for_removal
tags = post.tag_array.select { |tag| Danbooru.config.remove_tag_after_replacement?(tag) }
tags = tags.map { |tag| "-#{tag}" }
tags.join(" ")
end
end
include PresenterMethods
extend SearchMethods
end

406
app/models/upload.rb
View File

@@ -3,18 +3,67 @@ require "tmpdir"
class Upload < ApplicationRecord
class Error < Exception ; end
attr_accessor :file, :image_width, :image_height, :file_ext, :md5,
:file_size, :as_pending, :artist_commentary_title,
:artist_commentary_desc, :include_artist_commentary,
:referer_url, :downloaded_source, :replaced_post
belongs_to :uploader, :class_name => "User"
class Validator < ActiveModel::Validator
def validate(record)
if record.new_record?
validate_md5_uniqueness(record)
validate_video_duration(record)
end
validate_resolution(record)
end
def validate_md5_uniqueness(record)
if record.md5.nil?
return
end
md5_post = Post.find_by_md5(record.md5)
if md5_post.nil?
return
end
if record.replaced_post && record.replaced_post == md5_post
return
end
record.errors[:md5] << "duplicate: #{md5_post.id}"
end
def validate_resolution(record)
resolution = record.image_width.to_i * record.image_height.to_i
if resolution > Danbooru.config.max_image_resolution
record.errors[:base] << "image resolution is too large (resolution: #{(resolution / 1_000_000.0).round(1)} megapixels (#{record.image_width}x#{record.image_height}); max: #{Danbooru.config.max_image_resolution / 1_000_000} megapixels)"
end
end
def validate_video_duration(record)
if record.is_video? && record.video.duration > 120
record.errors[:base] << "video must not be longer than 2 minutes"
end
end
end
attr_accessor :as_pending,
:referer_url, :downloaded_source, :replaced_post, :file
belongs_to :uploader, :class_name => "User"
belongs_to :post, optional: true
before_validation :initialize_attributes
validate :uploader_is_not_limited, :on => :create
validate :file_or_source_is_present, :on => :create
validate :rating_given
before_validation :assign_rating_from_tags
validate :uploader_is_not_limited, on: :create
# validates :source, format: { with: /\Ahttps?/ }, if: ->(record) {record.file.blank?}, on: :create
validates :image_height, numericality: { less_than_or_equal_to: Danbooru.config.max_image_height }, allow_nil: true
validates :image_width, numericality: { less_than_or_equal_to: Danbooru.config.max_image_width }, allow_nil: true
validates :rating, inclusion: { in: %w(q e s) }, allow_nil: true
validates :md5, confirmation: true
validates :file_ext, format: { with: /jpg|gif|png|swf|webm|mp4|zip/ }, allow_nil: true
validates_with Validator
serialize :context, JSON
after_create {|rec| rec.uploader.increment!(:post_upload_count)}
def initialize_attributes
self.uploader_id = CurrentUser.user.id
@@ -22,189 +71,6 @@ class Upload < ApplicationRecord
self.server = Danbooru.config.server_host
end
module ValidationMethods
def uploader_is_not_limited
if !uploader.can_upload?
self.errors.add(:uploader, uploader.upload_limited_reason)
return false
else
return true
end
end
def file_or_source_is_present
if file.blank? && source.blank?
self.errors.add(:base, "Must choose file or specify source")
return false
else
return true
end
end
# Because uploads are processed serially, there's no race condition here.
def validate_md5_uniqueness
md5_post = Post.find_by_md5(md5)
if md5_post && replaced_post
raise "duplicate: #{md5_post.id}" if replaced_post != md5_post
elsif md5_post
raise "duplicate: #{md5_post.id}"
end
end
def validate_file_content_type
unless is_valid_content_type?
raise "invalid content type (only JPEG, PNG, GIF, SWF, MP4, and WebM files are allowed)"
end
if is_ugoira? && ugoira_service.empty?
raise "missing frame data for ugoira"
end
end
def validate_md5_confirmation
if !md5_confirmation.blank? && md5_confirmation != md5
raise "md5 mismatch"
end
end
def validate_dimensions
resolution = image_width * image_height
if resolution > Danbooru.config.max_image_resolution
raise "image resolution is too large (resolution: #{(resolution / 1_000_000.0).round(1)} megapixels (#{image_width}x#{image_height}); max: #{Danbooru.config.max_image_resolution / 1_000_000} megapixels)"
elsif image_width > Danbooru.config.max_image_width
raise "image width is too large (width: #{image_width}; max width: #{Danbooru.config.max_image_width})"
elsif image_height > Danbooru.config.max_image_height
raise "image height is too large (height: #{image_height}; max height: #{Danbooru.config.max_image_height})"
end
end
def rating_given
if rating.present?
return true
elsif tag_string =~ /(?:\s|^)rating:([qse])/i
self.rating = $1.downcase
return true
else
self.errors.add(:base, "Must specify a rating")
return false
end
end
def automatic_tags
return "" unless Danbooru.config.enable_dimension_autotagging
tags = []
tags << "video_with_sound" if is_video_with_audio?
tags << "animated_gif" if is_animated_gif?
tags << "animated_png" if is_animated_png?
tags.join(" ")
end
def validate_video_duration
unless uploader.is_admin?
if is_video? && video.duration > 120
raise "video must not be longer than 2 minutes"
end
end
end
end
module ConversionMethods
def process_upload
begin
update_attribute(:status, "processing")
self.source = source.to_s.strip
if is_downloadable?
self.downloaded_source, self.source, self.file = download_from_source(source, referer_url)
elsif self.file.respond_to?(:tempfile)
self.file = self.file.tempfile
end
self.file_ext = file_header_to_file_ext(file)
self.file_size = file.size
self.md5 = Digest::MD5.file(file.path).hexdigest
validate_file_content_type
validate_md5_uniqueness
validate_md5_confirmation
validate_video_duration
self.tag_string = "#{tag_string} #{automatic_tags}"
self.image_width, self.image_height = calculate_dimensions
validate_dimensions
save
end
end
def create_post_from_upload
post = convert_to_post
distribute_files(post)
if post.save
create_artist_commentary(post) if include_artist_commentary?
ugoira_service.save_frame_data(post) if is_ugoira?
notify_cropper(post)
update_attributes(:status => "completed", :post_id => post.id)
else
update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
end
post
end
def distribute_files(post)
preview_file, sample_file = generate_resizes
post.distribute_files(file, sample_file, preview_file)
ensure
preview_file.try(:close!)
sample_file.try(:close!)
end
def process!(force = false)
process_upload
post = create_post_from_upload
rescue Exception => x
update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
nil
ensure
file.try(:close!)
end
def ugoira_service
@ugoira_service ||= PixivUgoiraService.new
end
def convert_to_post
Post.new.tap do |p|
p.tag_string = tag_string
p.md5 = md5
p.file_ext = file_ext
p.image_width = image_width
p.image_height = image_height
p.rating = rating
p.source = source
p.file_size = file_size
p.uploader_id = uploader_id
p.uploader_ip_addr = uploader_ip_addr
p.parent_id = parent_id
if !uploader.can_upload_free? || upload_as_pending?
p.is_pending = true
end
end
end
def notify_cropper(post)
if ImageCropper.enabled?
# ImageCropper.notify(post)
end
end
end
module FileMethods
def is_image?
%w(jpg gif png).include?(file_ext)
@@ -218,120 +84,9 @@ class Upload < ApplicationRecord
%w(webm mp4).include?(file_ext)
end
def is_video_with_audio?
is_video? && video.audio_channels.present?
end
def is_ugoira?
%w(zip).include?(file_ext)
end
def is_animated_gif?
return false if file_ext != "gif"
# Check whether the gif has multiple frames by trying to load the second frame.
result = Vips::Image.gifload(file.path, page: 1) rescue $ERROR_INFO
if result.is_a?(Vips::Image)
true
elsif result.is_a?(Vips::Error) && result.message =~ /too few frames in GIF file/
false
else
raise result
end
end
def is_animated_png?
file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
end
end
module ResizerMethods
def generate_resizes
if is_video?
preview_file = generate_video_preview_for(video, Danbooru.config.small_image_width, Danbooru.config.small_image_width)
elsif is_ugoira?
preview_file = PixivUgoiraConverter.generate_preview(file)
sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data)
elsif is_image?
preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
if image_width > Danbooru.config.large_image_width
sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, image_height, 90)
end
end
[preview_file, sample_file]
end
def generate_video_preview_for(video, width, height)
dimension_ratio = video.width.to_f / video.height
if dimension_ratio > 1
height = (width / dimension_ratio).to_i
else
width = (height * dimension_ratio).to_i
end
output_file = Tempfile.new(binmode: true)
video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
output_file
end
end
module DimensionMethods
# Figures out the dimensions of the image.
def calculate_dimensions
if is_video?
[video.width, video.height]
elsif is_ugoira?
ugoira_service.calculate_dimensions(file.path)
[ugoira_service.width, ugoira_service.height]
else
image_size = ImageSpec.new(file.path)
[image_size.width, image_size.height]
end
end
end
module ContentTypeMethods
def is_valid_content_type?
file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/
end
def file_header_to_file_ext(file)
case File.read(file.path, 16)
when /^\xff\xd8/n
"jpg"
when /^GIF87a/, /^GIF89a/
"gif"
when /^\x89PNG\r\n\x1a\n/n
"png"
when /^CWS/, /^FWS/, /^ZWS/
"swf"
when /^\x1a\x45\xdf\xa3/n
"webm"
when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
"mp4"
when /^PK\x03\x04/
"zip"
else
"bin"
end
end
end
module DownloaderMethods
# Determines whether the source is downloadable
def is_downloadable?
source =~ /^https?:\/\// && file.blank?
end
def download_from_source(source, referer_url = nil)
download = Downloads::File.new(source, referer_url: referer_url)
file = download.download!
ugoira_service.load(download.data)
[download.downloaded_source, download.source, file]
end
end
module StatusMethods
@@ -347,10 +102,22 @@ class Upload < ApplicationRecord
status == "completed"
end
def is_preprocessed?
status == "preprocessed"
end
def is_preprocessing?
status == "preprocessing"
end
def is_duplicate?
status =~ /duplicate/
end
def is_errored?
status =~ /error:/
end
def duplicate_post_id
@duplicate_post_id ||= status[/duplicate: (\d+)/, 1]
end
@@ -448,28 +215,27 @@ class Upload < ApplicationRecord
end
end
module ArtistCommentaryMethods
def create_artist_commentary(post)
post.create_artist_commentary(
:original_title => artist_commentary_title,
:original_description => artist_commentary_desc
)
end
end
include ConversionMethods
include ValidationMethods
include FileMethods
include ResizerMethods
include DimensionMethods
include ContentTypeMethods
include DownloaderMethods
include StatusMethods
include UploaderMethods
include VideoMethods
extend SearchMethods
include ApiMethods
include ArtistCommentaryMethods
def uploader_is_not_limited
if !uploader.can_upload?
self.errors.add(:uploader, uploader.upload_limited_reason)
return false
else
return true
end
end
def assign_rating_from_tags
if tag_string =~ /(?:\s|^)rating:([qse])/i
self.rating = $1.downcase
end
end
def presenter
@presenter ||= UploadPresenter.new(self)
@@ -478,8 +244,4 @@ class Upload < ApplicationRecord
def upload_as_pending?
as_pending.to_s.truthy?
end
def include_artist_commentary?
include_artist_commentary.to_s.truthy?
end
end

66
app/views/uploads/new.html.erb
View File

@@ -21,6 +21,7 @@
<%= hidden_field_tag :url, params[:url] %>
<%= hidden_field_tag :ref, params[:ref] %>
<%= hidden_field_tag :normalized_url, @normalized_url %>
<%= f.hidden_field :md5_confirmation %>
<%= f.hidden_field :referer_url, :value => @source.try(:referer_url) %>
<% if CurrentUser.can_upload_free? %>
@@ -32,11 +33,15 @@
</div>
<% end %>
<div class="input">
<div class="input fallback">
<%= f.label :file %>
<%= f.file_field :file, :size => 50 %>
</div>
<div class="input" id="filedropzone">
<div class="placeholder"><span>Drag and drop a file here</span></div>
</div>
<div class="input">
<%= f.label :source %>
<% if params[:url].present? %>
@@ -106,15 +111,15 @@
<div class="input">
<div>
<%= f.label :tag_string, "Tags" %>
<%= f.text_area :tag_string, :size => "60x5", :data => { :autocomplete => "tag-edit" } %>
<%= f.text_area :tag_string, :size => "60x5", :spellcheck => false, :data => { :autocomplete => "tag-edit" } %>
<span id="open-edit-dialog" class="ui-icon ui-icon-arrow-1-ne" title="detach" style="display: none;"/>
</div>
<%= button_tag "Related tags", :id => "related-tags-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %>
<% TagCategory.related_button_list.each do |category| %>
<%= button_tag "#{TagCategory.related_button_mapping[category]}", :id => "related-#{category}-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %>
<% end %>
<% TagCategory.related_button_list.each do |category| %>
<%= button_tag "#{TagCategory.related_button_mapping[category]}", :id => "related-#{category}-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %>
<% end %>
</div>
<div class="input">
@@ -143,4 +148,55 @@
Upload - <%= Danbooru.config.app_name %>
<% end %>
<% content_for(:html_header) do %>
<script async src="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.4.0/min/dropzone.min.js"></script>
<script async src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.9-1/crypto-js.min.js"></script>
<script>
$(function() {
var enabled = true;
if (!window.FileReader) {
enabled = false;
}
if (!enabled) {
$("#filedropzone").remove();
return;
}
$("#filedropzone").dropzone({
paramName: "file",
url: "/uploads/preprocess",
createImageThumbnails: false,
addRemoveLinks: false,
maxFiles: 1,
acceptedFiles: "image/jpeg,image/png,image/gif",
previewTemplate: '<div class="dz-preview dz-file-preview"><div class="dz-details"><div class="dz-filename"><span data-dz-name></span></div><div class="dz-size" data-dz-size></div></div><div class="dz-progress"><span class="dz-upload" data-dz-uploadprogress></span></div><div class="dz-error-message"><span data-dz-errormessage></span></div></div>',
init: function() {
$(".fallback").hide();
this.on("drop", function(event) {
$("#filedropzone .placeholder").hide();
});
this.on("complete", function(file) {
$("#filedropzone .dz-progress").hide();
});
this.on("addedfile", function(file) {
var reader = new FileReader()
reader.addEventListener("loadend", function() {
$("#upload_md5_confirmation").val(CryptoJS.MD5(CryptoJS.enc.Latin1.parse(this.result)).toString());
});
reader.readAsBinaryString(file);
});
this.on("success", function(file) {
$("#filedropzone").addClass("success");
});
this.on("error", function(file, msg) {
$("#filedropzone").addClass("error");
});
}
});
});
</script>
<% end %>
<%= render "uploads/secondary_links" %>

4
app/views/uploads/show.html.erb
View File

@@ -12,7 +12,7 @@
<p>This upload has finished processing. <%= link_to "View the post", post_path(@upload.post_id) %>.</p>
<% elsif @upload.is_pending? %>
<p>This upload is waiting to be processed. Please wait a few seconds.</p>
<% elsif @upload.is_processing? %>
<% elsif @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
<p>This upload is being processed. Please wait a few seconds.</p>
<% elsif @upload.is_duplicate? %>
<p>This upload is a duplicate: <%= link_to "post ##{@upload.duplicate_post_id}", post_path(@upload.duplicate_post_id) %></p>
@@ -42,7 +42,7 @@
Upload - <%= Danbooru.config.app_name %>
<% end %>
<% if @upload.is_pending? || @upload.is_processing? %>
<% if @upload.is_pending? || @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
<% content_for(:html_header) do %>
<meta http-equiv="refresh" content="2">
<% end %>

3
config/initializers/ffmpeg.rb Normal file
View File

@@ -0,0 +1,3 @@
unless Rails.env.development?
FFMPEG.logger.level = Logger::ERROR
end

1
config/routes.rb
View File

@@ -279,6 +279,7 @@ Rails.application.routes.draw do
resource :tag_implication_request, :only => [:new, :create]
resources :uploads do
collection do
post :preprocess
get :batch
get :image_proxy
end

12
db/migrate/20180517190048_add_missing_fields_to_uploads.rb Normal file
View File

@@ -0,0 +1,12 @@
class AddMissingFieldsToUploads < ActiveRecord::Migration[5.2]
def change
add_column :uploads, :md5, :string
add_column :uploads, :file_ext, :string
add_column :uploads, :file_size, :integer
add_column :uploads, :image_width, :integer
add_column :uploads, :image_height, :integer
add_column :uploads, :artist_commentary_desc, :text
add_column :uploads, :artist_commentary_title, :text
add_column :uploads, :include_artist_commentary, :boolean
end
end

5
db/migrate/20180518175154_add_context_to_uploads.rb Normal file
View File

@@ -0,0 +1,5 @@
class AddContextToUploads < ActiveRecord::Migration[5.2]
def change
add_column :uploads, :context, :text
end
end

8
test/factories/upload.rb
View File

@@ -14,6 +14,14 @@ FactoryBot.define do
source "http://www.google.com/intl/en_ALL/images/logo.gif"
end
factory(:ugoira_upload) do
file do
f = Tempfile.new
IO.copy_stream("#{Rails.root}/test/fixtures/ugoira.zip", f.path)
ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "ugoira.zip")
end
end
factory(:jpg_upload) do
file do
f = Tempfile.new

BIN
test/files/valid_ugoira.zip Normal file
View File

Binary file not shown.

17
test/functional/uploads_controller_test.rb
View File

@@ -38,6 +38,15 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
assert_response :success
end
context "with a url" do
should "preprocess" do
assert_difference(-> { Upload.count }) do
get_auth new_upload_path, @user, params: {:url => "http://www.google.com/intl/en_ALL/images/logo.gif"}
assert_response :success
end
end
end
context "for a twitter post" do
should "render" do
skip "Twitter keys are not set" unless Danbooru.config.twitter_api_key
@@ -49,13 +58,15 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
context "for a post that has already been uploaded" do
setup do
as_user do
@post = create(:post, :source => "aaa")
@post = create(:post, :source => "http://google.com/aaa")
end
end
should "initialize the post" do
get_auth new_upload_path, @user, params: {:url => "http://google.com/aaa"}
assert_response :success
assert_difference(-> { Upload.count }, 0) do
get_auth new_upload_path, @user, params: {:url => "http://google.com/aaa"}
assert_response :success
end
end
end
end

907
test/models/upload_service_test.rb Normal file
View File

@@ -0,0 +1,907 @@
require 'test_helper'
class UploadServiceTest < ActiveSupport::TestCase
UGOIRA_CONTEXT = {
"ugoira" => {
"frame_data" => [
{"file" => "000001.jpg", "delay" => 200},
{"file" => "000002.jpg", "delay" => 200},
{"file" => "000003.jpg", "delay" => 200},
{"file" => "000004.jpg", "delay" => 200},
{"file" => "000005.jpg", "delay" => 250}
],
"content_type" => "image/jpeg"
}
}.freeze
context "::Utils" do
subject { UploadService::Utils }
context ".calculate_ugoira_dimensions" do
context "for a valid ugoira file" do
setup do
@path = "test/files/valid_ugoira.zip"
end
should "extract the dimensions" do
w, h = subject.calculate_ugoira_dimensions(@path)
assert_operator(w, :>, 0)
assert_operator(h, :>, 0)
end
end
context "for an invalid ugoira file" do
setup do
@path = "test/files/invalid_ugoira.zip"
end
should "raise an error" do
assert_raises(ImageSpec::Error) do
subject.calculate_ugoira_dimensions(@path)
end
end
end
end
context ".calculate_dimensions" do
context "for an ugoira" do
setup do
@file = File.open("test/files/valid_ugoira.zip", "rb")
@upload = mock()
@upload.stubs(:is_video?).returns(false)
@upload.stubs(:is_ugoira?).returns(true)
end
teardown do
@file.close
end
should "return the dimensions" do
subject.expects(:calculate_ugoira_dimensions).once.returns([60, 60])
subject.calculate_dimensions(@upload, @file) do |w, h|
assert_operator(w, :>, 0)
assert_operator(h, :>, 0)
end
end
end
context "for a video" do
setup do
@file = File.open("test/files/test-300x300.mp4", "rb")
@upload = mock()
@upload.stubs(:is_video?).returns(true)
end
teardown do
@file.close
end
should "return the dimensions" do
subject.calculate_dimensions(@upload, @file) do |w, h|
assert_operator(w, :>, 0)
assert_operator(h, :>, 0)
end
end
end
context "for an image" do
setup do
@file = File.open("test/files/test.jpg", "rb")
@upload = mock()
@upload.stubs(:is_video?).returns(false)
@upload.stubs(:is_ugoira?).returns(false)
end
teardown do
@file.close
end
should "find the dimensions" do
subject.calculate_dimensions(@upload, @file) do |w, h|
assert_operator(w, :>, 0)
assert_operator(h, :>, 0)
end
end
end
end
context ".process_file" do
setup do
@upload = FactoryBot.build(:jpg_upload)
@file = @upload.file
end
should "run" do
subject.expects(:distribute_files).twice
subject.process_file(@upload, @file)
assert_equal("jpg", @upload.file_ext)
assert_equal(28086, @upload.file_size)
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
assert_equal(335, @upload.image_height)
assert_equal(500, @upload.image_width)
end
end
context ".generate_resizes" do
context "for an ugoira" do
setup do
context = UGOIRA_CONTEXT
@file = File.open("test/fixtures/ugoira.zip", "rb")
@upload = mock()
@upload.stubs(:is_video?).returns(false)
@upload.stubs(:is_ugoira?).returns(true)
@upload.stubs(:context).returns(context)
end
should "generate a preview and a video" do
preview, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(sample.path), :>, 0)
preview.close
preview.unlink
sample.close
sample.unlink
end
end
context "for a video" do
teardown do
@file.close
end
context "for an mp4" do
setup do
@file = File.open("test/files/test-300x300.mp4", "rb")
@upload = mock()
@upload.stubs(:is_video?).returns(true)
@upload.stubs(:is_ugoira?).returns(false)
end
should "generate a video" do
preview, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
preview.close
preview.unlink
end
end
context "for a webm" do
setup do
@file = File.open("test/files/test-512x512.webm", "rb")
@upload = mock()
@upload.stubs(:is_video?).returns(true)
@upload.stubs(:is_ugoira?).returns(false)
end
should "generate a video" do
preview, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
preview.close
preview.unlink
end
end
end
context "for an image" do
teardown do
@file.close
end
setup do
@upload = mock()
@upload.stubs(:is_video?).returns(false)
@upload.stubs(:is_ugoira?).returns(false)
@upload.stubs(:is_image?).returns(true)
@upload.stubs(:image_width).returns(1200)
@upload.stubs(:image_height).returns(200)
end
context "for a jpeg" do
setup do
@file = File.open("test/files/test.jpg", "rb")
end
should "generate a preview" do
preview, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(sample.path), :>, 0)
preview.close
preview.unlink
sample.close
sample.unlink
end
end
context "for a png" do
setup do
@file = File.open("test/files/test.png", "rb")
end
should "generate a preview" do
preview, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(sample.path), :>, 0)
preview.close
preview.unlink
sample.close
sample.unlink
end
end
context "for a gif" do
setup do
@file = File.open("test/files/test.png", "rb")
end
should "generate a preview" do
preview, sample = subject.generate_resizes(@file, @upload)
assert_operator(File.size(preview.path), :>, 0)
assert_operator(File.size(sample.path), :>, 0)
preview.close
preview.unlink
sample.close
sample.unlink
end
end
end
end
context ".generate_video_preview_for" do
context "for an mp4" do
setup do
@path = "test/files/test-300x300.mp4"
@video = FFMPEG::Movie.new(@path)
end
should "generate a video" do
sample = subject.generate_video_preview_for(@video, 100, 100)
assert_operator(File.size(sample.path), :>, 0)
sample.close
sample.unlink
end
end
context "for a webm" do
setup do
@path = "test/files/test-512x512.webm"
@video = FFMPEG::Movie.new(@path)
end
should "generate a video" do
sample = subject.generate_video_preview_for(@video, 100, 100)
assert_operator(File.size(sample.path), :>, 0)
sample.close
sample.unlink
end
end
end
end
context "::Preprocessor" do
subject { UploadService::Preprocessor }
context "#download_from_source" do
setup do
@jpeg = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg"
@ugoira = "https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip"
end
should "work on a jpeg" do
file = subject.new({}).download_from_source(@jpeg) do |context|
assert_not_nil(context[:downloaded_source])
assert_not_nil(context[:source])
end
assert_operator(File.size(file.path), :>, 0)
file.close
end
should "work on an ugoira url" do
file = subject.new({}).download_from_source(@ugoira, referer_url: "https://www.pixiv.net") do |context|
assert_not_nil(context[:downloaded_source])
assert_not_nil(context[:source])
assert_not_nil(context[:ugoira])
end
assert_operator(File.size(file.path), :>, 0)
file.close
end
end
context "#start!" do
setup do
CurrentUser.user = travel_to(1.month.ago) do
FactoryBot.create(:user)
end
CurrentUser.ip_addr = "127.0.0.1"
@jpeg = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg"
@ugoira = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364"
@video = "https://www.sample-videos.com/video/mp4/720/big_buck_bunny_720p_1mb.mp4"
end
teardown do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
should "work for a jpeg" do
@service = subject.new(source: @jpeg)
@upload = @service.start!(CurrentUser.id)
assert_equal("preprocessed", @upload.status)
assert_not_nil(@upload.md5)
assert_equal("jpg", @upload.file_ext)
assert_operator(@upload.file_size, :>, 0)
assert_not_nil(@upload.source)
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :original)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :large)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :preview)))
end
should "work for an ugoira" do
@service = subject.new(source: @ugoira)
@upload = @service.start!(CurrentUser.id)
assert_equal("preprocessed", @upload.status)
assert_not_nil(@upload.md5)
assert_equal("zip", @upload.file_ext)
assert_operator(@upload.file_size, :>, 0)
assert_not_nil(@upload.source)
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "zip", :original)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "zip", :large)))
end
should "work for a video" do
@service = subject.new(source: @video)
@upload = @service.start!(CurrentUser.id)
assert_equal("preprocessed", @upload.status)
assert_not_nil(@upload.md5)
assert_equal("mp4", @upload.file_ext)
assert_operator(@upload.file_size, :>, 0)
assert_not_nil(@upload.source)
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "mp4", :original)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "mp4", :preview)))
end
context "on timeout errors" do
setup do
HTTParty.stubs(:get).raises(Net::ReadTimeout)
end
should "leave the upload in an error state" do
@service = subject.new(source: @video)
@upload = @service.start!(CurrentUser.id)
assert_match(/error:/, @upload.status)
end
end
end
end
context "::Replacer" do
context "for a file replacement" do
setup do
@new_file = upload_file("test/files/test.jpg")
@old_file = upload_file("test/files/test.png")
travel_to(1.month.ago) do
@user = FactoryBot.create(:user)
end
as_user do
@post = FactoryBot.create(:post, md5: Digest::MD5.hexdigest(@old_file.read))
@old_md5 = @post.md5
@post.stubs(:queue_delete_files)
@replacement = FactoryBot.create(:post_replacement, post: @post, replacement_url: "", replacement_file: @new_file)
end
end
subject { UploadService::Replacer.new(post: @post, replacement: @replacement) }
context "#process!" do
should "create a new upload" do
assert_difference(-> { Upload.count }) do
as_user { subject.process! }
end
end
should "create a comment" do
assert_difference(-> { @post.comments.count }) do
as_user { subject.process! }
@post.reload
end
end
should "not create a new post" do
assert_difference(-> { Post.count }, 0) do
as_user { subject.process! }
end
end
should "update the post's MD5" do
assert_changes(-> { @post.md5 }) do
as_user { subject.process! }
@post.reload
end
end
should "preserve the old values" do
as_user { subject.process! }
assert_equal(1500, @replacement.image_width_was)
assert_equal(1000, @replacement.image_height_was)
assert_equal(2000, @replacement.file_size_was)
assert_equal("jpg", @replacement.file_ext_was)
assert_equal(@old_md5, @replacement.md5_was)
end
should "record the new values" do
as_user { subject.process! }
assert_equal(500, @replacement.image_width)
assert_equal(335, @replacement.image_height)
assert_equal(28086, @replacement.file_size)
assert_equal("jpg", @replacement.file_ext)
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @replacement.md5)
end
should "correctly update the attributes" do
as_user { subject.process! }
assert_equal(500, @post.image_width)
assert_equal(335, @post.image_height)
assert_equal(28086, @post.file_size)
assert_equal("jpg", @post.file_ext)
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @post.md5)
assert(File.exists?(@post.file.path))
end
end
context "a post with the same file" do
should "not raise a duplicate error" do
upload_file("test/files/test.png") do |file|
assert_nothing_raised do
as_user { @post.replace!(replacement_file: file, replacement_url: "") }
end
end
end
should "not queue a deletion or log a comment" do
upload_file("test/files/test.png") do |file|
assert_no_difference(-> { @post.comments.count }) do
as_user { @post.replace!(replacement_file: file, replacement_url: "") }
@post.reload
end
end
end
end
end
context "for a source replacement" do
setup do
@new_url = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg"
travel_to(1.month.ago) do
@user = FactoryBot.create(:user)
end
as_user do
@post = FactoryBot.create(:post, uploader_ip_addr: "127.0.0.2")
@post.stubs(:queue_delete_files)
@replacement = FactoryBot.create(:post_replacement, post: @post, replacement_url: @new_url)
end
end
subject { UploadService::Replacer.new(post: @post, replacement: @replacement) }
context "a post when given a final_source" do
should "change the source to the final_source" do
replacement_url = "http://data.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_orwwptNBCE1wsfqepo1_raw.png"
final_source = "https://noizave.tumblr.com/post/162094447052"
as_user { @post.replace!(replacement_url: replacement_url, final_source: final_source) }
assert_equal(final_source, @post.source)
end
end
context "a post when replaced with a HTML source" do
should "record the image URL as the replacement URL, not the HTML source" do
skip "Twitter key not set" unless Danbooru.config.twitter_api_key
replacement_url = "https://twitter.com/nounproject/status/540944400767922176"
image_url = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:orig"
as_user { @post.replace!(replacement_url: replacement_url) }
assert_equal(image_url, @post.replacements.last.replacement_url)
end
end
context "#undo!" do
setup do
@user = travel_to(1.month.ago) { FactoryBot.create(:user) }
as_user do
@post = FactoryBot.create(:post, source: "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png")
@post.stubs(:queue_delete_files)
@post.replace!(replacement_url: "https://danbooru.donmai.us/data/preview/download.png", tags: "-tag1 tag2")
end
@replacement = @post.replacements.last
end
should "update the attributes" do
as_user do
subject.undo!
end
assert_equal("lowres tag2", @post.tag_string)
assert_equal(272, @post.image_width)
assert_equal(92, @post.image_height)
assert_equal(5969, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", @post.md5)
assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png", @post.source)
end
end
context "#process!" do
should "create a new upload" do
assert_difference(-> { Upload.count }) do
as_user { subject.process! }
end
end
should "create a comment" do
assert_difference(-> { @post.comments.count }) do
as_user { subject.process! }
@post.reload
end
end
should "not create a new post" do
assert_difference(-> { Post.count }, 0) do
as_user { subject.process! }
end
end
should "update the post's MD5" do
assert_changes(-> { @post.md5 }) do
as_user { subject.process! }
@post.reload
end
end
should "update the post's source" do
assert_changes(-> { @post.source }, nil, from: @post.source, to: @new_url) do
as_user { subject.process! }
@post.reload
end
end
should "not change the post status or uploader" do
assert_no_changes(-> { {ip_addr: @post.uploader_ip_addr.to_s, uploader: @post.uploader_id, pending: @post.is_pending?} }) do
as_user { subject.process! }
@post.reload
end
end
should "leave a system comment" do
as_user { subject.process! }
comment = @post.comments.last
assert_not_nil(comment)
assert_equal(User.system.id, comment.creator_id)
assert_match(/replaced this post/, comment.body)
end
end
context "a post with a pixiv html source" do
setup do
Delayed::Worker.delay_jobs = true
end
teardown do
Delayed::Worker.delay_jobs = false
end
should "replace with the full size image" do
begin
as_user do
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
end
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(16275, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
should "delete the old files after thirty days" do
begin
@post.unstub(:queue_delete_files)
FileUtils.expects(:rm_f).times(3)
as_user { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") }
travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do
Delayed::Worker.new.work_off
end
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "a post that is replaced by a ugoira" do
should "save the frame data" do
skip "ffmpeg not installed" unless check_ffmpeg
begin
as_user { @post.replace!(replacement_url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") }
@post.reload
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(2804, @post.file_size)
assert_equal("zip", @post.file_ext)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source)
assert_equal([{"delay"=>125, "file"=>"000000.jpg"}, {"delay"=>125,"file"=>"000001.jpg"}], @post.pixiv_ugoira_frame_data.data)
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "a post that is replaced to another file then replaced back to the original file" do
setup do
Delayed::Worker.delay_jobs = true
end
teardown do
Delayed::Worker.delay_jobs = false
end
should "not delete the original files" do
begin
FileUtils.expects(:rm_f).never
as_user do
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post.reload
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.reload
Upload.destroy_all
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
end
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do
Delayed::Worker.new.work_off
end
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "two posts that have had their files swapped" do
setup do
Delayed::Worker.delay_jobs = true
as_user do
@post1 = FactoryBot.create(:post)
@post2 = FactoryBot.create(:post)
end
end
teardown do
Delayed::Worker.delay_jobs = false
end
should "not delete the still active files" do
# swap the images between @post1 and @post2.
begin
as_user do
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.google.com/intl/en_ALL/images/logo.gif")
Upload.destroy_all
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
end
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
assert_nothing_raised { @post1.file(:original) }
assert_nothing_raised { @post2.file(:original) }
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "a post with notes" do
setup do
Note.any_instance.stubs(:merge_version?).returns(false)
as_user do
@post.update(image_width: 160, image_height: 164)
@note = @post.notes.create(x: 80, y: 82, width: 80, height: 82, body: "test")
@note.reload
end
end
should "rescale the notes" do
assert_equal([80, 82, 80, 82], [@note.x, @note.y, @note.width, @note.height])
begin
assert_difference(-> { @note.versions.count }) do
# replacement image is 80x82, so we're downscaling by 50% (160x164 -> 80x82).
as_user do
@post.replace!(replacement_url: "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg")
end
@note.reload
end
assert_equal([1024, 768, 1024, 768], [@note.x, @note.y, @note.width, @note.height])
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
end
end
context "#start!" do
subject { UploadService }
setup do
@source = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg"
CurrentUser.user = travel_to(1.month.ago) do
FactoryBot.create(:user)
end
CurrentUser.ip_addr = "127.0.0.1"
end
teardown do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
context "automatic tagging" do
setup do
@build_service = ->(file) { subject.new(file: file)}
end
should "tag animated png files" do
service = @build_service.call(upload_file("test/files/apng/normal_apng.png"))
upload = service.start!
assert_match(/animated_png/, upload.tag_string)
end
should "tag animated gif files" do
service = @build_service.call(upload_file("test/files/test-animated-86x52.gif"))
upload = service.start!
assert_match(/animated_gif/, upload.tag_string)
end
should "not tag static gif files" do
service = @build_service.call(upload_file("test/files/test-static-32x32.gif"))
upload = service.start!
assert_no_match(/animated_gif/, upload.tag_string)
end
end
context "that is too large" do
setup do
Danbooru.config.stubs(:max_image_resolution).returns(31*31)
end
should "should fail validation" do
service = subject.new(file: upload_file("test/files/test-static-32x32.gif"))
upload = service.start!
assert_match(/image resolution is too large/, upload.status)
end
end
context "with a preprocessing predecessor" do
setup do
@predecessor = FactoryBot.create(:source_upload, status: "preprocessing", source: @source, image_height: 0, image_width: 0, file_ext: "jpg")
Delayed::Worker.delay_jobs = true
end
teardown do
Delayed::Worker.delay_jobs = false
end
should "schedule a job later" do
service = subject.new(source: @source)
assert_difference(-> { Delayed::Job.count }) do
predecessor = service.start!
assert_equal(@predecessor, predecessor)
end
end
end
context "with a preprocessed predecessor" do
setup do
@predecessor = FactoryBot.create(:source_upload, status: "preprocessed", source: @source, file_size: 0, md5: "something", image_height: 0, image_width: 0, file_ext: "jpg")
@tags = 'hello world'
end
should "update the predecessor" do
service = subject.new(source: @source, tag_string: @tags)
predecessor = service.start!
assert_equal(@predecessor, predecessor)
assert_equal(@tags, predecessor.tag_string.strip)
end
end
context "with no predecessor" do
should "create an upload" do
service = subject.new(source: @source)
assert_difference(-> { Upload.count }) do
service.start!
end
end
end
end
context "#create_post_from_upload" do
subject { UploadService }
setup do
CurrentUser.user = travel_to(1.month.ago) do
FactoryBot.create(:user)
end
CurrentUser.ip_addr = "127.0.0.1"
end
teardown do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
context "for an ugoira" do
setup do
@upload = FactoryBot.create(:ugoira_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, context: UGOIRA_CONTEXT)
end
should "create a post" do
assert_difference(-> { PixivUgoiraFrameData.count }) do
post = subject.new({}).create_post_from_upload(@upload)
assert_equal([], post.errors.full_messages)
assert_not_nil(post.id)
end
end
end
context "for an image" do
setup do
@upload = FactoryBot.create(:source_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100)
end
should "create a commentary record" do
assert_difference(-> { ArtistCommentary.count }) do
subject.new({include_artist_commentary: true, artist_commentary_title: "blah", artist_commentary_desc: "blah"}).create_post_from_upload(@upload)
end
end
should "create a post" do
post = subject.new({}).create_post_from_upload(@upload)
assert_equal([], post.errors.full_messages)
assert_not_nil(post.id)
end
end
end
end

273
test/unit/post_replacement_test.rb
View File

@@ -27,278 +27,11 @@ class PostReplacementTest < ActiveSupport::TestCase
context "Replacing" do
setup do
CurrentUser.scoped(@uploader, "127.0.0.2") do
upload = FactoryBot.create(:jpg_upload, as_pending: "0", tag_string: "lowres tag1")
upload.process!
attributes = FactoryBot.attributes_for(:jpg_upload, as_pending: "0", tag_string: "lowres tag1")
service = UploadService.new(attributes)
upload = service.start!
@post = upload.post
end
end
context "a post from a generic source" do
setup do
@post.update(source: "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png")
@post.replace!(replacement_url: "https://www.google.com/intl/en_ALL/images/logo.gif", tags: "-tag1 tag2")
@replacement = @post.replacements.last
@upload = Upload.last
end
context "that is then undone" do
setup do
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
@replacement = @post.replacements.first
@replacement.undo!
@post.reload
end
should "update the attributes" do
assert_equal("lowres tag2", @post.tag_string)
assert_equal(272, @post.image_width)
assert_equal(92, @post.image_height)
assert_equal(5969, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", @post.md5)
assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png", @post.source)
end
end
should "create a post replacement record" do
assert_equal(@post.id, PostReplacement.last.post_id)
end
should "record the old file metadata" do
assert_equal(500, @replacement.image_width_was)
assert_equal(335, @replacement.image_height_was)
assert_equal(28086, @replacement.file_size_was)
assert_equal("jpg", @replacement.file_ext_was)
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @replacement.md5_was)
end
should "record the new file metadata" do
assert_equal(276, @replacement.image_width)
assert_equal(110, @replacement.image_height)
assert_equal(8558, @replacement.file_size)
assert_equal("gif", @replacement.file_ext)
assert_equal("e80d1c59a673f560785784fb1ac10959", @replacement.md5)
end
should "correctly update the attributes" do
assert_equal(@post.id, @upload.post.id)
assert_equal("completed", @upload.status)
assert_equal(276, @post.image_width)
assert_equal(110, @post.image_height)
assert_equal(8558, @post.file_size)
assert_equal("gif", @post.file_ext)
assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5)
assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source)
end
should "not change the post status or uploader" do
assert_equal("127.0.0.2", @post.uploader_ip_addr.to_s)
assert_equal(@uploader.id, @post.uploader_id)
assert_equal(false, @post.is_pending)
end
should "leave a system comment" do
comment = @post.comments.last
assert_not_nil(comment)
assert_equal(User.system.id, comment.creator_id)
assert_match(/replaced this post/, comment.body)
end
should "not send an @mention to the replacer" do
assert_equal(0, @replacer.dmails.size)
end
end
context "a post with notes" do
setup do
@post.update(image_width: 160, image_height: 164)
CurrentUser.scoped(@uploader, "127.0.0.1") do
@note = @post.notes.create(x: 80, y: 82, width: 80, height: 82, body: "test")
end
end
should "rescale the notes" do
assert_equal([80, 82, 80, 82], [@note.x, @note.y, @note.width, @note.height])
begin
assert_difference("@replacer.note_versions.count") do
# replacement image is 80x82, so we're downscaling by 50% (160x164 -> 80x82).
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@note.reload
end
assert_equal([40, 41, 40, 41], [@note.x, @note.y, @note.width, @note.height])
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "a post with a pixiv html source" do
should "replace with the full size image" do
begin
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(16275, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
should "delete the old files after thirty days" do
begin
old_file_path, old_preview_file_path = @post.file(:original).path, @post.file(:preview).path
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert(File.exists?(old_file_path))
assert(File.exists?(old_preview_file_path))
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
assert_not(File.exists?(old_file_path))
assert_not(File.exists?(old_preview_file_path))
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "a post that is replaced by a ugoira" do
should "save the frame data" do
skip "ffmpeg not installed" unless check_ffmpeg
begin
@post.replace!(replacement_url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.reload
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(2804, @post.file_size)
assert_equal("zip", @post.file_ext)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source)
assert_equal([{"delay"=>125, "file"=>"000001.jpg"}, {"delay"=>125,"file"=>"000002.jpg"}], @post.pixiv_ugoira_frame_data.data)
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "a post that is replaced to another file then replaced back to the original file" do
should "not delete the original files" do
skip "ffmpeg is not installed" unless check_ffmpeg
begin
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "two posts that have had their files swapped" do
should "not delete the still active files" do
skip "ffmpeg is not installed" unless check_ffmpeg
@post1 = FactoryBot.create(:post)
@post2 = FactoryBot.create(:post)
# swap the images between @post1 and @post2.
begin
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.google.com/intl/en_ALL/images/logo.gif")
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
assert_nothing_raised { @post1.file(:original) }
assert_nothing_raised { @post2.file(:original) }
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "a post with an uploaded file" do
should "work" do
upload_file("test/files/test.png") do |file|
@post.replace!(replacement_file: file, replacement_url: "")
assert_equal(@post.md5, Digest::MD5.file(file.tempfile).hexdigest)
assert_equal("file://test.png", @post.replacements.last.replacement_url)
end
end
end
context "a post when given a final_source" do
should "change the source to the final_source" do
replacement_url = "http://data.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_orwwptNBCE1wsfqepo1_raw.png"
final_source = "https://noizave.tumblr.com/post/162094447052"
@post.replace!(replacement_url: replacement_url, final_source: final_source)
assert_equal(final_source, @post.source)
end
end
context "a post when replaced with a HTML source" do
should "record the image URL as the replacement URL, not the HTML source" do
skip "Twitter key not set" unless Danbooru.config.twitter_api_key
replacement_url = "https://twitter.com/nounproject/status/540944400767922176"
image_url = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:orig"
@post.replace!(replacement_url: replacement_url)
assert_equal(image_url, @post.replacements.last.replacement_url)
end
end
context "a post with the same file" do
should "not raise a duplicate error" do
upload_file("test/files/test.jpg") do |file|
assert_nothing_raised do
@post.replace!(replacement_file: file, replacement_url: "")
end
end
end
should "not queue a deletion or log a comment" do
upload_file("test/files/test.jpg") do |file|
assert_no_difference(["@post.comments.count"]) do
@post.replace!(replacement_file: file, replacement_url: "")
end
end
end
end
end
end

18
test/unit/post_test.rb
View File

@@ -28,8 +28,7 @@ class PostTest < ActiveSupport::TestCase
context "Deletion:" do
context "Expunging a post" do
setup do
@upload = FactoryBot.create(:jpg_upload)
@upload.process!
@upload = UploadService.new(FactoryBot.attributes_for(:jpg_upload)).start!
@post = @upload.post
Favorite.add(post: @post, user: @user)
end
@@ -2677,4 +2676,19 @@ class PostTest < ActiveSupport::TestCase
end
end
end
context "#replace!" do
subject { @post.replace!(tags: "something", replacement_url: "https://danbooru.donmai.us/data/preview/download.png") }
setup do
@post = FactoryBot.create(:post)
@post.stubs(:queue_delete_files)
end
should "update the post" do
assert_changes(-> { @post.md5 }) do
subject
end
end
end
end

292
test/unit/upload_test.rb
View File

@@ -1,6 +1,8 @@
require 'test_helper'
class UploadTest < ActiveSupport::TestCase
SOURCE_URL = "https://upload.wikimedia.org/wikipedia/commons/thumb/6/66/NAMA_Machine_d%27Anticyth%C3%A8re_1.jpg/538px-NAMA_Machine_d%27Anticyth%C3%A8re_1.jpg?download"
context "In all cases" do
setup do
mock_iqdb_service!
@@ -28,298 +30,12 @@ class UploadTest < ActiveSupport::TestCase
end
end
context "image size calculator" do
should "discover the dimensions for a compressed SWF" do
@upload = FactoryBot.create(:upload, file: upload_file("test/files/compressed.swf"))
assert_equal([607, 756], @upload.calculate_dimensions)
end
should "discover the dimensions for a JPG with JFIF data" do
@upload = FactoryBot.create(:jpg_upload)
assert_equal([500, 335], @upload.calculate_dimensions)
end
should "discover the dimensions for a JPG with EXIF data" do
@upload = FactoryBot.create(:upload, file: upload_file("test/files/test-exif-small.jpg"))
assert_equal([529, 600], @upload.calculate_dimensions)
end
should "discover the dimensions for a JPG with no header data" do
@upload = FactoryBot.create(:upload, file: upload_file("test/files/test-blank.jpg"))
assert_equal([668, 996], @upload.calculate_dimensions)
end
should "discover the dimensions for a PNG" do
@upload = FactoryBot.create(:upload, file: upload_file("test/files/test.png"))
assert_equal([768, 1024], @upload.calculate_dimensions)
end
should "discover the dimensions for a GIF" do
@upload = FactoryBot.create(:upload, file: upload_file("test/files/test.gif"))
assert_equal([400, 400], @upload.calculate_dimensions)
end
end
context "content type calculator" do
should "know how to parse jpeg, png, gif, and swf file headers" do
@upload = FactoryBot.create(:jpg_upload)
assert_equal("jpg", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.jpg")))
assert_equal("gif", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.gif")))
assert_equal("png", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.png")))
assert_equal("swf", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/compressed.swf")))
assert_equal("bin", @upload.file_header_to_file_ext(File.open("#{Rails.root}/README.md")))
end
end
context "downloader" do
context "for a zip that is not an ugoira" do
should "not validate" do
@upload = FactoryBot.create(:upload, file: upload_file("test/files/invalid_ugoira.zip"))
@upload.process!
assert_equal("error: RuntimeError - missing frame data for ugoira", @upload.status)
end
end
context "that is a pixiv ugoira" do
setup do
@url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654"
@upload = FactoryBot.create(:source_upload, :source => @url, :tag_string => "ugoira")
end
should "process successfully" do
begin
_, _, output_file = @upload.download_from_source(@url, "")
assert_operator(output_file.size, :>, 1_000)
assert_equal("zip", @upload.file_header_to_file_ext(output_file))
rescue Net::OpenTimeout
skip "Remote connection to #{@url} failed"
end
end
end
end
context "determining if a file is downloadable" do
should "classify HTTP sources as downloadable" do
@upload = FactoryBot.create(:source_upload, :source => "http://www.google.com/1.jpg")
assert_not_nil(@upload.is_downloadable?)
end
should "classify HTTPS sources as downloadable" do
@upload = FactoryBot.create(:source_upload, :source => "https://www.google.com/1.jpg")
assert_not_nil(@upload.is_downloadable?)
end
should "classify non-HTTP/HTTPS sources as not downloadable" do
@upload = FactoryBot.create(:source_upload, :source => "ftp://www.google.com/1.jpg")
assert_nil(@upload.is_downloadable?)
end
end
context "file processor" do
should "parse and process a cgi file representation" do
@upload = FactoryBot.create(:upload, file: upload_file("test/files/test.jpg"))
assert_nothing_raised {@upload.process_upload}
assert_equal(28086, @upload.file_size)
end
should "process a transparent png" do
@upload = FactoryBot.create(:upload, file: upload_file("test/files/alpha.png"))
assert_nothing_raised {@upload.process_upload}
assert_equal(1136, @upload.file_size)
end
end
context "hash calculator" do
should "caculate the hash" do
@upload = FactoryBot.create(:jpg_upload)
@upload.process_upload
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
end
end
context "resizer" do
should "generate several resized versions of the image" do
@upload = FactoryBot.create(:upload, file_ext: "jpg", image_width: 1356, image_height: 911, file: upload_file("test/files/test-large.jpg"))
preview_file, sample_file = @upload.generate_resizes
assert_operator(preview_file.size, :>, 1_000)
assert_operator(sample_file.size, :>, 1_000)
end
end
should "increment the uploaders post_upload_count" do
@upload = FactoryBot.create(:source_upload)
assert_difference("CurrentUser.user.post_upload_count", 1) do
@upload.process!
assert_difference(-> { CurrentUser.user.post_upload_count }) do
FactoryBot.create(:source_upload)
CurrentUser.user.reload
end
end
context "with an artist commentary" do
setup do
@upload = FactoryBot.create(:source_upload,
include_artist_commentary: "1",
artist_commentary_title: "",
artist_commentary_desc: "blah",
)
end
should "create an artist commentary when processed" do
assert_difference("ArtistCommentary.count") do
@upload.process!
end
end
end
should "process completely for a downloaded image" do
@upload = FactoryBot.create(:source_upload,
:rating => "s",
:uploader_ip_addr => "127.0.0.1",
:tag_string => "hoge foo"
)
assert_difference("Post.count") do
assert_nothing_raised {@upload.process!}
end
post = Post.last
assert_equal("http://www.google.com/intl/en_ALL/images/logo.gif", post.source)
assert_equal("foo hoge lowres", post.tag_string)
assert_equal("s", post.rating)
assert_equal(@upload.uploader_id, post.uploader_id)
assert_equal("127.0.0.1", post.uploader_ip_addr.to_s)
assert_equal(@upload.md5, post.md5)
assert_equal("gif", post.file_ext)
assert_equal(276, post.image_width)
assert_equal(110, post.image_height)
assert_equal(8558, post.file_size)
assert_equal(post.id, @upload.post_id)
assert_equal("completed", @upload.status)
end
context "automatic tagging" do
should "tag animated png files" do
@upload = FactoryBot.build(:upload, file_ext: "png", file: upload_file("test/files/apng/normal_apng.png"))
assert_equal("animated_png", @upload.automatic_tags)
end
should "tag animated gif files" do
@upload = FactoryBot.build(:upload, file_ext: "gif", file: upload_file("test/files/test-animated-86x52.gif"))
assert_equal("animated_gif", @upload.automatic_tags)
end
should "not tag static gif files" do
@upload = FactoryBot.build(:upload, file_ext: "gif", file: upload_file("test/files/test-static-32x32.gif"))
assert_equal("", @upload.automatic_tags)
end
end
context "that is too large" do
should "should fail validation" do
Danbooru.config.stubs(:max_image_resolution).returns(31*31)
@upload = FactoryBot.create(:upload, file: upload_file("test/files/test-static-32x32.gif"))
@upload.process!
assert_match(/image resolution is too large/, @upload.status)
end
end
end
should "process completely for a pixiv ugoira" do
skip "ffmpeg is not installed" unless check_ffmpeg
@upload = FactoryBot.create(:source_upload, source: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654")
assert_difference(["PixivUgoiraFrameData.count", "Post.count"]) do
@upload.process!
assert_equal([], @upload.errors.full_messages)
end
post = Post.last
assert_not_nil(post.pixiv_ugoira_frame_data)
assert_equal("0d94800c4b520bf3d8adda08f95d31e2", post.md5)
assert_equal(60, post.image_width)
assert_equal(60, post.image_height)
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2014/10/05/23/42/23/46378654_ugoira1920x1080.zip", post.source)
assert_nothing_raised { post.file(:original) }
assert_nothing_raised { post.file(:preview) }
end
should "process completely for an uploaded image" do
@upload = FactoryBot.create(:jpg_upload,
:rating => "s",
:uploader_ip_addr => "127.0.0.1",
:tag_string => "hoge foo",
:file => upload_file("test/files/test.jpg"),
)
assert_difference("Post.count") do
assert_nothing_raised {@upload.process!}
end
post = Post.last
assert_equal("foo hoge lowres", post.tag_string)
assert_equal("s", post.rating)
assert_equal(@upload.uploader_id, post.uploader_id)
assert_equal("127.0.0.1", post.uploader_ip_addr.to_s)
assert_equal(@upload.md5, post.md5)
assert_equal("jpg", post.file_ext)
assert_nothing_raised { post.file(:original) }
assert_equal(28086, post.file(:original).size)
assert_equal(post.id, @upload.post_id)
assert_equal("completed", @upload.status)
end
should "process completely for a .webm" do
upload = FactoryBot.create(:upload, rating: "s", file: upload_file("test/files/test-512x512.webm"))
assert_difference("Post.count") do
upload.process!
assert_equal("completed", upload.status)
end
post = Post.last
assert_includes(post.tag_array, "webm")
assert_equal("webm", upload.file_ext)
assert_equal(12345, upload.file_size)
assert_equal(512, upload.image_width)
assert_equal(512, upload.image_height)
assert_equal("34dd2489f7aaa9e57eda1b996ff26ff7", upload.md5)
assert_nothing_raised { post.file(:preview) }
assert_nothing_raised { post.file(:original) }
end
should "process completely for a .mp4" do
upload = FactoryBot.create(:upload, rating: "s", file: upload_file("test/files/test-300x300.mp4"))
assert_difference("Post.count") do
upload.process!
assert_equal("completed", upload.status)
end
post = Post.last
assert_includes(post.tag_array, "mp4")
assert_equal("mp4", upload.file_ext)
assert_equal(18677, upload.file_size)
assert_equal(300, upload.image_width)
assert_equal(300, upload.image_height)
assert_equal("865c93102cad3e8a893d6aac6b51b0d2", upload.md5)
assert_nothing_raised { post.file(:preview) }
assert_nothing_raised { post.file(:original) }
end
should "process completely for a null source" do
@upload = FactoryBot.create(:jpg_upload, :source => nil)
assert_difference("Post.count") do
assert_nothing_raised {@upload.process!}
end
end
context "on timeout errors" do
should "leave the upload in an error state" do
HTTParty.stubs(:get).raises(Net::ReadTimeout)
@upload = FactoryBot.create(:source_upload)
@upload.process!
assert_match(/\Aerror/, @upload.status)
end
end
end
end