diff --git a/Procfile b/Procfile index 5a2f07d60..11556b87b 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ unicorn: bundle exec rails server -jobs: bundle exec script/delayed_job run +jobs: bundle exec script/delayed_job run diff --git a/app/assets/javascripts/related_tag.js.erb b/app/assets/javascripts/related_tag.js.erb index 88ea5ed71..af359d4e7 100644 --- a/app/assets/javascripts/related_tag.js.erb +++ b/app/assets/javascripts/related_tag.js.erb @@ -34,7 +34,6 @@ $dest.empty(); Danbooru.RelatedTag.build_recent_and_frequent($dest); $dest.append("Loading..."); - $("#related-tags-container").show(); $.get("/related_tag.json", { "query": Danbooru.RelatedTag.current_tag(), "category": category @@ -93,6 +92,9 @@ } Danbooru.RelatedTag.process_response = function(data) { + if (data.tags.length || data.wiki_page_tags.length || data.other_wikis.length) { + $("#related-tags-container").show(); + } Danbooru.RelatedTag.recent_search = data; Danbooru.RelatedTag.build_all(); } diff --git a/app/assets/javascripts/uploads.js b/app/assets/javascripts/uploads.js index ad511a1b6..406739e4a 100644 --- a/app/assets/javascripts/uploads.js +++ b/app/assets/javascripts/uploads.js @@ -33,7 +33,7 @@ Danbooru.Upload.initialize_submit = function() { $("#form").submit(function(e) { var error_messages = []; - if (($("#upload_file").val() === "") && ($("#upload_source").val() === "")) { + if (($("#upload_file").val() === "") && ($("#upload_source").val() === "") && $("#upload_md5_confirmation").val() === "") { error_messages.push("Must choose file or specify source"); } if (!$("#upload_rating_s").prop("checked") && !$("#upload_rating_q").prop("checked") && !$("#upload_rating_e").prop("checked") && diff --git a/app/assets/stylesheets/specific/posts.scss b/app/assets/stylesheets/specific/posts.scss index d0b6ab846..af8cfe0a3 100644 --- a/app/assets/stylesheets/specific/posts.scss +++ b/app/assets/stylesheets/specific/posts.scss @@ -471,9 +471,10 @@ div#c-post-versions, div#c-artist-versions { div#c-posts, div#c-uploads { /* Fetch source data box */ div#source-info { + border-radius: 4px; margin: 1em 0; padding: 1em; - border: 1px solid gray; + border: 1px solid #666; p { margin: 0; diff --git a/app/assets/stylesheets/specific/related_tags.scss b/app/assets/stylesheets/specific/related_tags.scss index ff5fe72ee..6511d9ecd 100644 --- a/app/assets/stylesheets/specific/related_tags.scss +++ b/app/assets/stylesheets/specific/related_tags.scss @@ -1,6 +1,7 @@ @import "../common/000_vars.scss"; div#related-tags-container { + display: none; padding-right: 2em; h1 { @@ -14,6 +15,7 @@ div.related-tags { padding: 1em; background: #EEE; overflow: auto; + border-radius: 4px; div.tag-column { max-width: 15em; diff --git a/app/assets/stylesheets/specific/uploads.scss b/app/assets/stylesheets/specific/uploads.scss index 49b71d0b1..1cc7bece5 100644 --- a/app/assets/stylesheets/specific/uploads.scss +++ b/app/assets/stylesheets/specific/uploads.scss @@ -34,6 +34,48 @@ div#c-uploads { div.field_with_errors { display: inline; } + + #filedropzone { + border: 4px dashed #DDD; + padding: 0; + min-height: 100px; + display: flex; + align-items: center; + justify-content: center; + border-radius: 4px; + + .placeholder { + font-style: italic; + color: #333; + height: 100%; + } + + &.error { + border-color: darken(#f2dede, 30%); + background-color: #f2dede; + } + + &.success { + border-color: darken(#dff0d8, 30%); + background-color: #dff0d8; + } + } + + .dz-preview { + margin-bottom: 1em; + } + + .dz-progress { + height: 20px; + width: 300px; + border: 1px solid #CCC; + + .dz-upload { + background-color: #F5F5FF; + display: block; + height: 20px; + } + } } div#a-index { diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index 654d5ca42..e6afe9dbf 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -1,33 +1,19 @@ class UploadsController < ApplicationController before_action :member_only, except: [:index, :show] respond_to :html, :xml, :json, :js + skip_before_action :verify_authenticity_token, only: [:preprocess] def new - @upload = Upload.new @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first - if params[:url] - download = Downloads::File.new(params[:url]) - @normalized_url, _, _ = download.before_download(params[:url], {}) - @post = find_post_by_url(@normalized_url) - - begin - @source = Sources::Site.new(params[:url], :referer_url => params[:ref]) - @remote_size = download.size - rescue Exception - end - end + @upload, @post, @source, @normalized_url, @remote_size = UploadService::ControllerHelper.prepare( + url: params[:url], ref: params[:ref] + ) respond_with(@upload) end def batch @url = params.dig(:batch, :url) || params[:url] - @source = nil - - if @url - @source = Sources::Site.new(@url, :referer_url => params[:ref]) - @source.get - end - + @source = UploadService::ControllerHelper.batch(@url, params[:ref]) respond_with(@source) end @@ -56,15 +42,19 @@ class UploadsController < ApplicationController end end + def preprocess + @upload, @post, @source, @normalized_url, @remote_size = UploadService::ControllerHelper.prepare( + url: params[:url], file: params[:file], ref: params[:ref] + ) + render body: nil + end + def create - @upload = Upload.create(upload_params) + @service = UploadService.new(upload_params) + @upload = @service.start! - if @upload.errors.empty? - post = @upload.process! - - if post.present? && post.valid? && post.warnings.any? - flash[:notice] = post.warnings.full_messages.join(".\n \n") - end + if @service.warnings.any? + flash[:notice] = @service.warnings.join(".\n \n") end save_recent_tags @@ -73,14 +63,6 @@ class UploadsController < ApplicationController private - def find_post_by_url(normalized_url) - if normalized_url.nil? - Post.where("SourcePattern(lower(posts.source)) = ?", params[:url]).first - else - Post.where("SourcePattern(lower(posts.source)) IN (?)", [params[:url], @normalized_url]).first - end - end - def save_recent_tags if @upload tags = Tag.scan_tags(@upload.tag_string) @@ -94,7 +76,7 @@ class UploadsController < ApplicationController permitted_params = %i[ file source tag_string rating status parent_id artist_commentary_title artist_commentary_desc include_artist_commentary referer_url - md5_confirmation as_pending + md5_confirmation as_pending ] params.require(:upload).permit(permitted_params) diff --git a/app/logical/pixiv_ugoira_service.rb b/app/logical/pixiv_ugoira_service.rb deleted file mode 100644 index 58536fa0d..000000000 --- a/app/logical/pixiv_ugoira_service.rb +++ /dev/null @@ -1,33 +0,0 @@ -class PixivUgoiraService - attr_reader :width, :height, :frame_data, :content_type - - def save_frame_data(post) - PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id) - end - - def calculate_dimensions(source_path) - folder = Zip::File.new(source_path) - tempfile = Tempfile.new("ugoira-dimensions") - - begin - folder.first.extract(tempfile.path) {true} - image_size = ImageSpec.new(tempfile.path) - @width = image_size.width - @height = image_size.height - ensure - tempfile.close - tempfile.unlink - end - end - - def load(data) - if data[:is_ugoira] - @frame_data = data[:ugoira_frame_data] - @content_type = data[:ugoira_content_type] - end - end - - def empty? - @frame_data.nil? - end -end diff --git a/app/logical/upload_service.rb b/app/logical/upload_service.rb new file mode 100644 index 000000000..1db0597db --- /dev/null +++ b/app/logical/upload_service.rb @@ -0,0 +1,587 @@ +class UploadService + module ControllerHelper + def self.prepare(url: nil, file: nil, ref: nil) + upload = Upload.new + + if url + # this gets called from UploadsController#new so we need + # to preprocess async + Preprocessor.new(source: url).delay(queue: "default").start!(CurrentUser.user.id) + + download = Downloads::File.new(url) + normalized_url, _, _ = download.before_download(url, {}) + post = if normalized_url.nil? + Post.where("SourcePattern(lower(posts.source)) = ?", url).first + else + Post.where("SourcePattern(lower(posts.source)) IN (?)", [url, normalized_url]).first + end + + begin + source = Sources::Site.new(url, :referer_url => ref) + remote_size = download.size + rescue Exception + end + + return [upload, post, source, normalized_url, remote_size] + elsif file + # this gets called via XHR so we can process sync + Preprocessor.new(file: file).start!((CurrentUser.user.id)) + end + + return [upload] + end + + def self.batch(url, ref = nil) + if url + source = Sources::Site.new(url, :referer_url => ref) + source.get + return source + end + end + end + + module Utils + def self.file_header_to_file_ext(file) + case File.read(file.path, 16) + when /^\xff\xd8/n + "jpg" + when /^GIF87a/, /^GIF89a/ + "gif" + when /^\x89PNG\r\n\x1a\n/n + "png" + when /^CWS/, /^FWS/, /^ZWS/ + "swf" + when /^\x1a\x45\xdf\xa3/n + "webm" + when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/ + "mp4" + when /^PK\x03\x04/ + "zip" + else + "bin" + end + end + + def self.delete_file(md5, file_ext) + if Post.where(md5: md5).exists? + return + end + + Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :original) + Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :large) + Danbooru.config.storage_manager.delete_file(nil, md5, file_ext, :preview) + Danbooru.config.backup_storage_manager.delete_file(nil, md5, file_ext, :original) + Danbooru.config.backup_storage_manager.delete_file(nil, md5, file_ext, :large) + Danbooru.config.backup_storage_manager.delete_file(nil, md5, file_ext, :preview) + end + + def self.calculate_ugoira_dimensions(source_path) + folder = Zip::File.new(source_path) + Tempfile.open("ugoira-dim-") do |tempfile| + folder.first.extract(tempfile.path) { true } + image_size = ImageSpec.new(tempfile.path) + return [image_size.width, image_size.height] + end + end + + def self.calculate_dimensions(upload, file) + if upload.is_video? + video = FFMPEG::Movie.new(file.path) + yield(video.width, video.height) + + elsif upload.is_ugoira? + w, h = calculate_ugoira_dimensions(file.path) + yield(w, h) + + else + image_size = ImageSpec.new(file.path) + yield(image_size.width, image_size.height) + end + end + + def self.distribute_files(file, record, type) + [Danbooru.config.storage_manager, Danbooru.config.backup_storage_manager].each do |sm| + sm.store_file(file, record, type) + end + end + + def self.is_downloadable?(source) + source.match?(/^https?:\/\//) + end + + def self.generate_resizes(file, upload) + if upload.is_video? + video = FFMPEG::Movie.new(file.path) + preview_file = generate_video_preview_for(video, Danbooru.config.small_image_width, Danbooru.config.small_image_width) + + elsif upload.is_ugoira? + preview_file = PixivUgoiraConverter.generate_preview(file) + sample_file = PixivUgoiraConverter.generate_webm(file, upload.context["ugoira"]["frame_data"]) + + elsif upload.is_image? + preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) + + if upload.image_width > Danbooru.config.large_image_width + sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, upload.image_height, 90) + end + end + + [preview_file, sample_file] + end + + def self.generate_video_preview_for(video, width, height) + dimension_ratio = video.width.to_f / video.height + if dimension_ratio > 1 + height = (width / dimension_ratio).to_i + else + width = (height * dimension_ratio).to_i + end + + output_file = Tempfile.new(binmode: true) + video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) + output_file + end + + def self.process_file(upload, file) + upload.file = file + upload.file_ext = Utils.file_header_to_file_ext(file) + upload.file_size = file.size + upload.md5 = Digest::MD5.file(file.path).hexdigest + + if Post.where(md5: upload.md5).exists? + # abort early if this post already exists + raise Upload::Error.new("Post with MD5 #{upload.md5} already exists") + end + + Utils.calculate_dimensions(upload, file) do |width, height| + upload.image_width = width + upload.image_height = height + end + + upload.tag_string = "#{upload.tag_string} #{Utils.automatic_tags(upload, file)}" + + preview_file, sample_file = Utils.generate_resizes(file, upload) + + begin + Utils.distribute_files(file, upload, :original) + Utils.distribute_files(sample_file, upload, :large) if sample_file.present? + Utils.distribute_files(preview_file, upload, :preview) if preview_file.present? + ensure + preview_file.try(:close!) + sample_file.try(:close!) + end + + # in case this upload never finishes processing, we need to delete the + # distributed files in the future + Danbooru.config.other_server_hosts.each do |host| + UploadService::Utils.delay(queue: host, run_at: 10.minutes.from_now).delete_file(upload.md5, upload.file_ext) + end + end + + # these methods are only really used during upload processing even + # though logically they belong on upload. post can rely on the + # automatic tag that's added. + def self.is_animated_gif?(upload, file) + return false if upload.file_ext != "gif" + + # Check whether the gif has multiple frames by trying to load the second frame. + result = Vips::Image.gifload(file.path, page: 1) rescue $ERROR_INFO + if result.is_a?(Vips::Image) + true + elsif result.is_a?(Vips::Error) && result.message =~ /too few frames in GIF file/ + false + else + raise result + end + end + + def self.is_animated_png?(upload, file) + upload.file_ext == "png" && APNGInspector.new(file.path).inspect!.animated? + end + + def self.is_video_with_audio?(upload, file) + video = FFMPEG::Movie.new(file.path) + upload.is_video? && video.audio_channels.present? + end + + def self.automatic_tags(upload, file) + return "" unless Danbooru.config.enable_dimension_autotagging + + tags = [] + tags << "video_with_sound" if is_video_with_audio?(upload, file) + tags << "animated_gif" if is_animated_gif?(upload, file) + tags << "animated_png" if is_animated_png?(upload, file) + tags.join(" ") + end + end + + class Preprocessor + attr_reader :params + + def initialize(params) + @params = params + end + + def source + params[:source] + end + + def md5 + params[:md5_confirmation] + end + + def in_progress? + if source.present? + Upload.where(status: "preprocessing", source: source).exists? + elsif md5.present? + Upload.where(status: "preprocessing", md5: md5).exists? + else + false + end + end + + def predecessor + if source.present? + Upload.where(status: ["preprocessed", "preprocessing"], source: source).first + elsif md5.present? + Upload.where(status: ["preprocessed", "preprocessing"], md5: md5).first + end + end + + def completed? + predecessor.present? + end + + def start!(uploader_id) + if source.present? + if !Utils.is_downloadable?(source) + return + end + + if Post.where(source: source).exists? + return + end + + if Upload.where(source: source, status: "completed").exists? + return + end + + if Upload.where(source: source).where("status like ?", "error%").exists? + return + end + end + + params[:rating] ||= "q" + params[:tag_string] ||= "tagme" + + CurrentUser.as(User.find(uploader_id)) do + upload = Upload.create!(params) + + upload.update(status: "preprocessing") + + begin + if source.present? + file = download_from_source(source, referer_url: upload.referer_url) do |context| + upload.downloaded_source = context[:downloaded_source] + upload.source = context[:source] + + if context[:ugoira] + upload.context = { ugoira: context[:ugoira] } + end + end + elsif params[:file].present? + file = params[:file] + end + + Utils.process_file(upload, file) + + upload.rating = params[:rating] + upload.tag_string = params[:tag_string] + upload.status = "preprocessed" + upload.save! + rescue Exception => x + upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n")) + end + + return upload + end + end + + def finish!(upload = nil) + pred = upload || self.predecessor() + pred.attributes = self.params + pred.status = "completed" + pred.save + return pred + end + + def download_from_source(source, referer_url: nil) + download = Downloads::File.new(source, referer_url: referer_url) + file = download.download! + context = { + downloaded_source: download.downloaded_source, + source: download.source + } + + if download.data[:is_ugoira] + context[:ugoira] = { + frame_data: download.data[:ugoira_frame_data], + content_type: download.data[:ugoira_content_type] + } + end + + yield(context) + + return file + end + end + + class Replacer + attr_reader :post, :replacement + + def initialize(post:, replacement:) + @post = post + @replacement = replacement + end + + def comment_replacement_message(post, replacement) + %("#{replacement.creator.name}":[/users/#{replacement.creator.id}] replaced this post with a new image:\n\n#{replacement_message(post, replacement)}) + end + + def replacement_message(post, replacement) + linked_source = linked_source(replacement.replacement_url) + linked_source_was = linked_source(post.source_was) + + <<-EOS.strip_heredoc + [table] + [tbody] + [tr] + [th]Old[/th] + [td]#{linked_source_was}[/td] + [td]#{post.md5_was}[/td] + [td]#{post.file_ext_was}[/td] + [td]#{post.image_width_was} x #{post.image_height_was}[/td] + [td]#{post.file_size_was.to_s(:human_size, precision: 4)}[/td] + [/tr] + [tr] + [th]New[/th] + [td]#{linked_source}[/td] + [td]#{post.md5}[/td] + [td]#{post.file_ext}[/td] + [td]#{post.image_width} x #{post.image_height}[/td] + [td]#{post.file_size.to_s(:human_size, precision: 4)}[/td] + [/tr] + [/tbody] + [/table] + EOS + end + + def linked_source(source) + return nil if source.nil? + + # truncate long sources in the middle: "www.pixiv.net...lust_id=23264933" + truncated_source = source.gsub(%r{\Ahttps?://}, "").truncate(64, omission: "...#{source.last(32)}") + + if source =~ %r{\Ahttps?://}i + %("#{truncated_source}":[#{source}]) + else + truncated_source + end + end + + def undo! + undo_replacement = post.replacements.create(replacement_url: replacement.original_url) + undoer = Replacer.new(post: post, replacement: undo_replacement) + undoer.process! + end + + def process! + preprocessor = Preprocessor.new( + rating: post.rating, + tag_string: replacement.tags, + source: replacement.replacement_url, + file: replacement.replacement_file + ) + upload = preprocessor.start!(CurrentUser.id) + return if upload.is_errored? + upload = preprocessor.finish!(upload) + return if upload.is_errored? + md5_changed = upload.md5 != post.md5 + + if replacement.replacement_file.present? + replacement.replacement_url = "file://#{replacement.replacement_file.original_filename}" + elsif upload.downloaded_source.present? + replacement.replacement_url = upload.downloaded_source + end + + if md5_changed + post.queue_delete_files(PostReplacement::DELETION_GRACE_PERIOD) + end + + replacement.file_ext = upload.file_ext + replacement.file_size = upload.file_size + replacement.image_height = upload.image_height + replacement.image_width = upload.image_width + replacement.md5 = upload.md5 + + post.md5 = upload.md5 + post.file_ext = upload.file_ext + post.image_width = upload.image_width + post.image_height = upload.image_height + post.file_size = upload.file_size + post.source = upload.downloaded_source || upload.source + post.tag_string = upload.tag_string + + update_ugoira_frame_data(post, upload) + + if md5_changed + CurrentUser.as(User.system) do + post.comments.create!(body: comment_replacement_message(post, replacement), do_not_bump_post: true) + end + end + + if replacement.final_source.present? + post.update(source: replacement.final_source) + end + + replacement.save! + post.save! + + rescale_notes(post) + end + + def rescale_notes(post) + x_scale = post.image_width.to_f / post.image_width_before_last_save.to_f + y_scale = post.image_height.to_f / post.image_height_before_last_save.to_f + + post.notes.each do |note| + note.reload + note.rescale!(x_scale, y_scale) + end + end + + def update_ugoira_frame_data(post, upload) + post.pixiv_ugoira_frame_data.destroy if post.pixiv_ugoira_frame_data.present? + + unless post.is_ugoira? + return + end + + PixivUgoiraFrameData.create( + post_id: post.id, + data: upload.context["ugoira"]["frame_data"], + content_type: upload.context["ugoira"]["content_type"] + ) + end + end + + attr_reader :params, :post, :upload + + def initialize(params) + @params = params + end + + def start! + preprocessor = Preprocessor.new(params) + + if preprocessor.in_progress? + delay(queue: "default", run_at: 5.seconds.from_now).start! + return preprocessor.predecessor + end + + if preprocessor.completed? + @upload = preprocessor.finish! + create_post_from_upload(@upload) + return @upload + end + + params[:rating] ||= "q" + params[:tag_string] ||= "tagme" + @upload = Upload.create!(params) + + begin + if @upload.invalid? + return @upload + end + + @upload.update(status: "processing") + + if @upload.file.present? + Utils.process_file(upload, @upload.file) + else + # sources will be handled in preprocessing now + end + + @upload.save! + @post = create_post_from_upload(@upload) + return @upload + + rescue Exception => x + @upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n")) + @upload + end + end + + def warnings + return [] if @post.nil? + return @post.warnings.full_messages + end + + def source + params[:source] + end + + def include_artist_commentary? + params[:include_artist_commentary].to_s.truthy? + end + + def create_post_from_upload(upload) + @post = convert_to_post(upload) + @post.save! + + upload.update(status: "error: " + @post.errors.full_messages.join(", ")) + + if upload.context && upload.context["ugoira"] + PixivUgoiraFrameData.create( + post_id: @post.id, + data: upload.context["ugoira"]["frame_data"], + content_type: upload.context["ugoira"]["content_type"] + ) + end + + if include_artist_commentary? + @post.create_artist_commentary( + :original_title => params[:artist_commentary_title], + :original_description => params[:artist_commentary_desc] + ) + end + + notify_cropper(@post) if ImageCropper.enabled? + upload.update(status: "completed", post_id: @post.id) + @post + end + + def convert_to_post(upload) + Post.new.tap do |p| + p.tag_string = upload.tag_string + p.md5 = upload.md5 + p.file_ext = upload.file_ext + p.image_width = upload.image_width + p.image_height = upload.image_height + p.rating = upload.rating + p.source = upload.source + p.file_size = upload.file_size + p.uploader_id = upload.uploader_id + p.uploader_ip_addr = upload.uploader_ip_addr + p.parent_id = upload.parent_id + + if !upload.uploader.can_upload_free? || upload.upload_as_pending? + p.is_pending = true + end + end + end + + def notify_cropper(post) + # ImageCropper.notify(post) + end +end diff --git a/app/models/post.rb b/app/models/post.rb index 70ab22905..a4b56dc88 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -1396,7 +1396,8 @@ class Post < ApplicationRecord def replace!(params) transaction do replacement = replacements.create(params) - replacement.process! + processor = UploadService::Replacer.new(post: self, replacement: replacement) + processor.process! replacement end end diff --git a/app/models/post_replacement.rb b/app/models/post_replacement.rb index 8ea4046ce..840a90f8b 100644 --- a/app/models/post_replacement.rb +++ b/app/models/post_replacement.rb @@ -18,166 +18,40 @@ class PostReplacement < ApplicationRecord self.md5_was = post.md5 end - def undo! - undo_replacement = post.replacements.create(replacement_url: original_url) - undo_replacement.process! - end - - def process! - upload = nil - - transaction do - upload = Upload.create!( - file: replacement_file, - source: replacement_url, - rating: post.rating, - tag_string: self.tags, - replaced_post: post, - ) - - upload.process_upload - upload.update(status: "completed", post_id: post.id) - md5_changed = (upload.md5 != post.md5) - - if replacement_file.present? - self.replacement_url = "file://#{replacement_file.original_filename}" - else - self.replacement_url = upload.downloaded_source + concerning :Search do + class_methods do + def post_tags_match(query) + PostQueryBuilder.new(query).build(self.joins(:post)) end - # queue the deletion *before* updating the post so that we use the old - # md5/file_ext to delete the old files. if saving the post fails, - # this is rolled back so the job won't run. - if md5_changed - post.queue_delete_files(DELETION_GRACE_PERIOD) - end + def search(params = {}) + q = super - self.file_ext = upload.file_ext - self.file_size = upload.file_size - self.image_height = upload.image_height - self.image_width = upload.image_width - self.md5 = upload.md5 - - post.md5 = upload.md5 - post.file_ext = upload.file_ext - post.image_width = upload.image_width - post.image_height = upload.image_height - post.file_size = upload.file_size - post.source = final_source.presence || upload.source - post.tag_string = upload.tag_string - - rescale_notes - update_ugoira_frame_data(upload) - - if md5_changed - CurrentUser.as(User.system) do - post.comments.create!(body: comment_replacement_message, do_not_bump_post: true) + if params[:creator_id].present? + q = q.where(creator_id: params[:creator_id].split(",").map(&:to_i)) end + + if params[:creator_name].present? + q = q.where(creator_id: User.name_to_id(params[:creator_name])) + end + + if params[:post_id].present? + q = q.where(post_id: params[:post_id].split(",").map(&:to_i)) + end + + if params[:post_tags_match].present? + q = q.post_tags_match(params[:post_tags_match]) + end + + q.apply_default_order(params) end - - save! - post.save! - end - - # point of no return: these things can't be rolled back, so we do them - # only after the transaction successfully commits. - upload.distribute_files(post) - post.update_iqdb_async - end - - def rescale_notes - x_scale = post.image_width.to_f / post.image_width_was.to_f - y_scale = post.image_height.to_f / post.image_height_was.to_f - - post.notes.each do |note| - note.rescale!(x_scale, y_scale) end end - def update_ugoira_frame_data(upload) - post.pixiv_ugoira_frame_data.destroy if post.pixiv_ugoira_frame_data.present? - upload.ugoira_service.save_frame_data(post) if post.is_ugoira? + def suggested_tags_for_removal + tags = post.tag_array.select { |tag| Danbooru.config.remove_tag_after_replacement?(tag) } + tags = tags.map { |tag| "-#{tag}" } + tags.join(" ") end - module SearchMethods - def post_tags_match(query) - PostQueryBuilder.new(query).build(self.joins(:post)) - end - - def search(params = {}) - q = super - - if params[:creator_id].present? - q = q.where(creator_id: params[:creator_id].split(",").map(&:to_i)) - end - - if params[:creator_name].present? - q = q.where(creator_id: User.name_to_id(params[:creator_name])) - end - - if params[:post_id].present? - q = q.where(post_id: params[:post_id].split(",").map(&:to_i)) - end - - if params[:post_tags_match].present? - q = q.post_tags_match(params[:post_tags_match]) - end - - q.apply_default_order(params) - end - end - - module PresenterMethods - def comment_replacement_message - %("#{creator.name}":[/users/#{creator.id}] replaced this post with a new image:\n\n#{replacement_message}) - end - - def replacement_message - linked_source = linked_source(replacement_url) - linked_source_was = linked_source(post.source_was) - - <<-EOS.strip_heredoc - [table] - [tbody] - [tr] - [th]Old[/th] - [td]#{linked_source_was}[/td] - [td]#{post.md5_was}[/td] - [td]#{post.file_ext_was}[/td] - [td]#{post.image_width_was} x #{post.image_height_was}[/td] - [td]#{post.file_size_was.to_s(:human_size, precision: 4)}[/td] - [/tr] - [tr] - [th]New[/th] - [td]#{linked_source}[/td] - [td]#{post.md5}[/td] - [td]#{post.file_ext}[/td] - [td]#{post.image_width} x #{post.image_height}[/td] - [td]#{post.file_size.to_s(:human_size, precision: 4)}[/td] - [/tr] - [/tbody] - [/table] - EOS - end - - def linked_source(source) - # truncate long sources in the middle: "www.pixiv.net...lust_id=23264933" - truncated_source = source.gsub(%r{\Ahttps?://}, "").truncate(64, omission: "...#{source.last(32)}") - - if source =~ %r{\Ahttps?://}i - %("#{truncated_source}":[#{source}]) - else - truncated_source - end - end - - def suggested_tags_for_removal - tags = post.tag_array.select { |tag| Danbooru.config.remove_tag_after_replacement?(tag) } - tags = tags.map { |tag| "-#{tag}" } - tags.join(" ") - end - end - - include PresenterMethods - extend SearchMethods end diff --git a/app/models/upload.rb b/app/models/upload.rb index ce107900f..ee6a1a3d6 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -3,18 +3,67 @@ require "tmpdir" class Upload < ApplicationRecord class Error < Exception ; end - attr_accessor :file, :image_width, :image_height, :file_ext, :md5, - :file_size, :as_pending, :artist_commentary_title, - :artist_commentary_desc, :include_artist_commentary, - :referer_url, :downloaded_source, :replaced_post - belongs_to :uploader, :class_name => "User" + class Validator < ActiveModel::Validator + def validate(record) + if record.new_record? + validate_md5_uniqueness(record) + validate_video_duration(record) + end + validate_resolution(record) + end + + def validate_md5_uniqueness(record) + if record.md5.nil? + return + end + + md5_post = Post.find_by_md5(record.md5) + + if md5_post.nil? + return + end + + if record.replaced_post && record.replaced_post == md5_post + return + end + + record.errors[:md5] << "duplicate: #{md5_post.id}" + end + + def validate_resolution(record) + resolution = record.image_width.to_i * record.image_height.to_i + + if resolution > Danbooru.config.max_image_resolution + record.errors[:base] << "image resolution is too large (resolution: #{(resolution / 1_000_000.0).round(1)} megapixels (#{record.image_width}x#{record.image_height}); max: #{Danbooru.config.max_image_resolution / 1_000_000} megapixels)" + end + end + + def validate_video_duration(record) + if record.is_video? && record.video.duration > 120 + record.errors[:base] << "video must not be longer than 2 minutes" + end + end + end + + + attr_accessor :as_pending, + :referer_url, :downloaded_source, :replaced_post, :file + belongs_to :uploader, :class_name => "User" belongs_to :post, optional: true before_validation :initialize_attributes - validate :uploader_is_not_limited, :on => :create - validate :file_or_source_is_present, :on => :create - validate :rating_given + before_validation :assign_rating_from_tags + validate :uploader_is_not_limited, on: :create + # validates :source, format: { with: /\Ahttps?/ }, if: ->(record) {record.file.blank?}, on: :create + validates :image_height, numericality: { less_than_or_equal_to: Danbooru.config.max_image_height }, allow_nil: true + validates :image_width, numericality: { less_than_or_equal_to: Danbooru.config.max_image_width }, allow_nil: true + validates :rating, inclusion: { in: %w(q e s) }, allow_nil: true + validates :md5, confirmation: true + validates :file_ext, format: { with: /jpg|gif|png|swf|webm|mp4|zip/ }, allow_nil: true + validates_with Validator + serialize :context, JSON + after_create {|rec| rec.uploader.increment!(:post_upload_count)} def initialize_attributes self.uploader_id = CurrentUser.user.id @@ -22,189 +71,6 @@ class Upload < ApplicationRecord self.server = Danbooru.config.server_host end - module ValidationMethods - def uploader_is_not_limited - if !uploader.can_upload? - self.errors.add(:uploader, uploader.upload_limited_reason) - return false - else - return true - end - end - - def file_or_source_is_present - if file.blank? && source.blank? - self.errors.add(:base, "Must choose file or specify source") - return false - else - return true - end - end - - # Because uploads are processed serially, there's no race condition here. - def validate_md5_uniqueness - md5_post = Post.find_by_md5(md5) - - if md5_post && replaced_post - raise "duplicate: #{md5_post.id}" if replaced_post != md5_post - elsif md5_post - raise "duplicate: #{md5_post.id}" - end - end - - def validate_file_content_type - unless is_valid_content_type? - raise "invalid content type (only JPEG, PNG, GIF, SWF, MP4, and WebM files are allowed)" - end - - if is_ugoira? && ugoira_service.empty? - raise "missing frame data for ugoira" - end - end - - def validate_md5_confirmation - if !md5_confirmation.blank? && md5_confirmation != md5 - raise "md5 mismatch" - end - end - - def validate_dimensions - resolution = image_width * image_height - - if resolution > Danbooru.config.max_image_resolution - raise "image resolution is too large (resolution: #{(resolution / 1_000_000.0).round(1)} megapixels (#{image_width}x#{image_height}); max: #{Danbooru.config.max_image_resolution / 1_000_000} megapixels)" - elsif image_width > Danbooru.config.max_image_width - raise "image width is too large (width: #{image_width}; max width: #{Danbooru.config.max_image_width})" - elsif image_height > Danbooru.config.max_image_height - raise "image height is too large (height: #{image_height}; max height: #{Danbooru.config.max_image_height})" - end - end - - def rating_given - if rating.present? - return true - elsif tag_string =~ /(?:\s|^)rating:([qse])/i - self.rating = $1.downcase - return true - else - self.errors.add(:base, "Must specify a rating") - return false - end - end - - def automatic_tags - return "" unless Danbooru.config.enable_dimension_autotagging - - tags = [] - tags << "video_with_sound" if is_video_with_audio? - tags << "animated_gif" if is_animated_gif? - tags << "animated_png" if is_animated_png? - tags.join(" ") - end - - def validate_video_duration - unless uploader.is_admin? - if is_video? && video.duration > 120 - raise "video must not be longer than 2 minutes" - end - end - end - end - - module ConversionMethods - def process_upload - begin - update_attribute(:status, "processing") - - self.source = source.to_s.strip - if is_downloadable? - self.downloaded_source, self.source, self.file = download_from_source(source, referer_url) - elsif self.file.respond_to?(:tempfile) - self.file = self.file.tempfile - end - - self.file_ext = file_header_to_file_ext(file) - self.file_size = file.size - self.md5 = Digest::MD5.file(file.path).hexdigest - - validate_file_content_type - validate_md5_uniqueness - validate_md5_confirmation - validate_video_duration - - self.tag_string = "#{tag_string} #{automatic_tags}" - self.image_width, self.image_height = calculate_dimensions - validate_dimensions - - save - end - end - - def create_post_from_upload - post = convert_to_post - distribute_files(post) - - if post.save - create_artist_commentary(post) if include_artist_commentary? - ugoira_service.save_frame_data(post) if is_ugoira? - notify_cropper(post) - update_attributes(:status => "completed", :post_id => post.id) - else - update_attribute(:status, "error: " + post.errors.full_messages.join(", ")) - end - - post - end - - def distribute_files(post) - preview_file, sample_file = generate_resizes - post.distribute_files(file, sample_file, preview_file) - ensure - preview_file.try(:close!) - sample_file.try(:close!) - end - - def process!(force = false) - process_upload - post = create_post_from_upload - rescue Exception => x - update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n")) - nil - ensure - file.try(:close!) - end - - def ugoira_service - @ugoira_service ||= PixivUgoiraService.new - end - - def convert_to_post - Post.new.tap do |p| - p.tag_string = tag_string - p.md5 = md5 - p.file_ext = file_ext - p.image_width = image_width - p.image_height = image_height - p.rating = rating - p.source = source - p.file_size = file_size - p.uploader_id = uploader_id - p.uploader_ip_addr = uploader_ip_addr - p.parent_id = parent_id - - if !uploader.can_upload_free? || upload_as_pending? - p.is_pending = true - end - end - end - - def notify_cropper(post) - if ImageCropper.enabled? - # ImageCropper.notify(post) - end - end - end - module FileMethods def is_image? %w(jpg gif png).include?(file_ext) @@ -218,120 +84,9 @@ class Upload < ApplicationRecord %w(webm mp4).include?(file_ext) end - def is_video_with_audio? - is_video? && video.audio_channels.present? - end - def is_ugoira? %w(zip).include?(file_ext) end - - def is_animated_gif? - return false if file_ext != "gif" - - # Check whether the gif has multiple frames by trying to load the second frame. - result = Vips::Image.gifload(file.path, page: 1) rescue $ERROR_INFO - if result.is_a?(Vips::Image) - true - elsif result.is_a?(Vips::Error) && result.message =~ /too few frames in GIF file/ - false - else - raise result - end - end - - def is_animated_png? - file_ext == "png" && APNGInspector.new(file.path).inspect!.animated? - end - end - - module ResizerMethods - def generate_resizes - if is_video? - preview_file = generate_video_preview_for(video, Danbooru.config.small_image_width, Danbooru.config.small_image_width) - elsif is_ugoira? - preview_file = PixivUgoiraConverter.generate_preview(file) - sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data) - elsif is_image? - preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85) - - if image_width > Danbooru.config.large_image_width - sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, image_height, 90) - end - end - - [preview_file, sample_file] - end - - def generate_video_preview_for(video, width, height) - dimension_ratio = video.width.to_f / video.height - if dimension_ratio > 1 - height = (width / dimension_ratio).to_i - else - width = (height * dimension_ratio).to_i - end - - output_file = Tempfile.new(binmode: true) - video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"}) - output_file - end - end - - module DimensionMethods - # Figures out the dimensions of the image. - def calculate_dimensions - if is_video? - [video.width, video.height] - elsif is_ugoira? - ugoira_service.calculate_dimensions(file.path) - [ugoira_service.width, ugoira_service.height] - else - image_size = ImageSpec.new(file.path) - [image_size.width, image_size.height] - end - end - end - - module ContentTypeMethods - def is_valid_content_type? - file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/ - end - - def file_header_to_file_ext(file) - case File.read(file.path, 16) - when /^\xff\xd8/n - "jpg" - when /^GIF87a/, /^GIF89a/ - "gif" - when /^\x89PNG\r\n\x1a\n/n - "png" - when /^CWS/, /^FWS/, /^ZWS/ - "swf" - when /^\x1a\x45\xdf\xa3/n - "webm" - when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/ - "mp4" - when /^PK\x03\x04/ - "zip" - else - "bin" - end - end - end - - module DownloaderMethods - # Determines whether the source is downloadable - def is_downloadable? - source =~ /^https?:\/\// && file.blank? - end - - def download_from_source(source, referer_url = nil) - download = Downloads::File.new(source, referer_url: referer_url) - file = download.download! - ugoira_service.load(download.data) - - [download.downloaded_source, download.source, file] - end end module StatusMethods @@ -347,10 +102,22 @@ class Upload < ApplicationRecord status == "completed" end + def is_preprocessed? + status == "preprocessed" + end + + def is_preprocessing? + status == "preprocessing" + end + def is_duplicate? status =~ /duplicate/ end + def is_errored? + status =~ /error:/ + end + def duplicate_post_id @duplicate_post_id ||= status[/duplicate: (\d+)/, 1] end @@ -448,28 +215,27 @@ class Upload < ApplicationRecord end end - module ArtistCommentaryMethods - def create_artist_commentary(post) - post.create_artist_commentary( - :original_title => artist_commentary_title, - :original_description => artist_commentary_desc - ) - end - end - - include ConversionMethods - include ValidationMethods include FileMethods - include ResizerMethods - include DimensionMethods - include ContentTypeMethods - include DownloaderMethods include StatusMethods include UploaderMethods include VideoMethods extend SearchMethods include ApiMethods - include ArtistCommentaryMethods + + def uploader_is_not_limited + if !uploader.can_upload? + self.errors.add(:uploader, uploader.upload_limited_reason) + return false + else + return true + end + end + + def assign_rating_from_tags + if tag_string =~ /(?:\s|^)rating:([qse])/i + self.rating = $1.downcase + end + end def presenter @presenter ||= UploadPresenter.new(self) @@ -478,8 +244,4 @@ class Upload < ApplicationRecord def upload_as_pending? as_pending.to_s.truthy? end - - def include_artist_commentary? - include_artist_commentary.to_s.truthy? - end end diff --git a/app/views/uploads/new.html.erb b/app/views/uploads/new.html.erb index d392321e8..c576152b8 100644 --- a/app/views/uploads/new.html.erb +++ b/app/views/uploads/new.html.erb @@ -21,6 +21,7 @@ <%= hidden_field_tag :url, params[:url] %> <%= hidden_field_tag :ref, params[:ref] %> <%= hidden_field_tag :normalized_url, @normalized_url %> + <%= f.hidden_field :md5_confirmation %> <%= f.hidden_field :referer_url, :value => @source.try(:referer_url) %> <% if CurrentUser.can_upload_free? %> @@ -32,11 +33,15 @@ <% end %> -
+
<%= f.label :file %> <%= f.file_field :file, :size => 50 %>
+
+
Drag and drop a file here
+
+
<%= f.label :source %> <% if params[:url].present? %> @@ -106,15 +111,15 @@
<%= f.label :tag_string, "Tags" %> - <%= f.text_area :tag_string, :size => "60x5", :data => { :autocomplete => "tag-edit" } %> + <%= f.text_area :tag_string, :size => "60x5", :spellcheck => false, :data => { :autocomplete => "tag-edit" } %>
<%= button_tag "Related tags", :id => "related-tags-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %> - <% TagCategory.related_button_list.each do |category| %> - <%= button_tag "#{TagCategory.related_button_mapping[category]}", :id => "related-#{category}-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %> - <% end %> + <% TagCategory.related_button_list.each do |category| %> + <%= button_tag "#{TagCategory.related_button_mapping[category]}", :id => "related-#{category}-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %> + <% end %>
@@ -143,4 +148,55 @@ Upload - <%= Danbooru.config.app_name %> <% end %> +<% content_for(:html_header) do %> + + + +<% end %> + <%= render "uploads/secondary_links" %> diff --git a/app/views/uploads/show.html.erb b/app/views/uploads/show.html.erb index b173ac7b7..8ffa690a1 100644 --- a/app/views/uploads/show.html.erb +++ b/app/views/uploads/show.html.erb @@ -12,7 +12,7 @@

This upload has finished processing. <%= link_to "View the post", post_path(@upload.post_id) %>.

<% elsif @upload.is_pending? %>

This upload is waiting to be processed. Please wait a few seconds.

- <% elsif @upload.is_processing? %> + <% elsif @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>

This upload is being processed. Please wait a few seconds.

<% elsif @upload.is_duplicate? %>

This upload is a duplicate: <%= link_to "post ##{@upload.duplicate_post_id}", post_path(@upload.duplicate_post_id) %>

@@ -42,7 +42,7 @@ Upload - <%= Danbooru.config.app_name %> <% end %> -<% if @upload.is_pending? || @upload.is_processing? %> +<% if @upload.is_pending? || @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %> <% content_for(:html_header) do %> <% end %> diff --git a/config/initializers/ffmpeg.rb b/config/initializers/ffmpeg.rb new file mode 100644 index 000000000..387508f8c --- /dev/null +++ b/config/initializers/ffmpeg.rb @@ -0,0 +1,3 @@ +unless Rails.env.development? + FFMPEG.logger.level = Logger::ERROR +end diff --git a/config/routes.rb b/config/routes.rb index b6d9c5584..2a0664b0b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -279,6 +279,7 @@ Rails.application.routes.draw do resource :tag_implication_request, :only => [:new, :create] resources :uploads do collection do + post :preprocess get :batch get :image_proxy end diff --git a/db/migrate/20180517190048_add_missing_fields_to_uploads.rb b/db/migrate/20180517190048_add_missing_fields_to_uploads.rb new file mode 100644 index 000000000..491d03243 --- /dev/null +++ b/db/migrate/20180517190048_add_missing_fields_to_uploads.rb @@ -0,0 +1,12 @@ +class AddMissingFieldsToUploads < ActiveRecord::Migration[5.2] + def change + add_column :uploads, :md5, :string + add_column :uploads, :file_ext, :string + add_column :uploads, :file_size, :integer + add_column :uploads, :image_width, :integer + add_column :uploads, :image_height, :integer + add_column :uploads, :artist_commentary_desc, :text + add_column :uploads, :artist_commentary_title, :text + add_column :uploads, :include_artist_commentary, :boolean + end +end diff --git a/db/migrate/20180518175154_add_context_to_uploads.rb b/db/migrate/20180518175154_add_context_to_uploads.rb new file mode 100644 index 000000000..d53bdddb4 --- /dev/null +++ b/db/migrate/20180518175154_add_context_to_uploads.rb @@ -0,0 +1,5 @@ +class AddContextToUploads < ActiveRecord::Migration[5.2] + def change + add_column :uploads, :context, :text + end +end diff --git a/test/factories/upload.rb b/test/factories/upload.rb index 7b886da5f..3c59f799d 100644 --- a/test/factories/upload.rb +++ b/test/factories/upload.rb @@ -14,6 +14,14 @@ FactoryBot.define do source "http://www.google.com/intl/en_ALL/images/logo.gif" end + factory(:ugoira_upload) do + file do + f = Tempfile.new + IO.copy_stream("#{Rails.root}/test/fixtures/ugoira.zip", f.path) + ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "ugoira.zip") + end + end + factory(:jpg_upload) do file do f = Tempfile.new diff --git a/test/files/valid_ugoira.zip b/test/files/valid_ugoira.zip new file mode 100644 index 000000000..463bed1c8 Binary files /dev/null and b/test/files/valid_ugoira.zip differ diff --git a/test/functional/uploads_controller_test.rb b/test/functional/uploads_controller_test.rb index 93502a8d2..16e5d64c7 100644 --- a/test/functional/uploads_controller_test.rb +++ b/test/functional/uploads_controller_test.rb @@ -38,6 +38,15 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest assert_response :success end + context "with a url" do + should "preprocess" do + assert_difference(-> { Upload.count }) do + get_auth new_upload_path, @user, params: {:url => "http://www.google.com/intl/en_ALL/images/logo.gif"} + assert_response :success + end + end + end + context "for a twitter post" do should "render" do skip "Twitter keys are not set" unless Danbooru.config.twitter_api_key @@ -49,13 +58,15 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest context "for a post that has already been uploaded" do setup do as_user do - @post = create(:post, :source => "aaa") + @post = create(:post, :source => "http://google.com/aaa") end end should "initialize the post" do - get_auth new_upload_path, @user, params: {:url => "http://google.com/aaa"} - assert_response :success + assert_difference(-> { Upload.count }, 0) do + get_auth new_upload_path, @user, params: {:url => "http://google.com/aaa"} + assert_response :success + end end end end diff --git a/test/models/upload_service_test.rb b/test/models/upload_service_test.rb new file mode 100644 index 000000000..12fd458dc --- /dev/null +++ b/test/models/upload_service_test.rb @@ -0,0 +1,907 @@ +require 'test_helper' + +class UploadServiceTest < ActiveSupport::TestCase + UGOIRA_CONTEXT = { + "ugoira" => { + "frame_data" => [ + {"file" => "000001.jpg", "delay" => 200}, + {"file" => "000002.jpg", "delay" => 200}, + {"file" => "000003.jpg", "delay" => 200}, + {"file" => "000004.jpg", "delay" => 200}, + {"file" => "000005.jpg", "delay" => 250} + ], + "content_type" => "image/jpeg" + } + }.freeze + + context "::Utils" do + subject { UploadService::Utils } + + context ".calculate_ugoira_dimensions" do + context "for a valid ugoira file" do + setup do + @path = "test/files/valid_ugoira.zip" + end + + should "extract the dimensions" do + w, h = subject.calculate_ugoira_dimensions(@path) + assert_operator(w, :>, 0) + assert_operator(h, :>, 0) + end + end + + context "for an invalid ugoira file" do + setup do + @path = "test/files/invalid_ugoira.zip" + end + + should "raise an error" do + assert_raises(ImageSpec::Error) do + subject.calculate_ugoira_dimensions(@path) + end + end + end + end + + context ".calculate_dimensions" do + context "for an ugoira" do + setup do + @file = File.open("test/files/valid_ugoira.zip", "rb") + @upload = mock() + @upload.stubs(:is_video?).returns(false) + @upload.stubs(:is_ugoira?).returns(true) + end + + teardown do + @file.close + end + + should "return the dimensions" do + subject.expects(:calculate_ugoira_dimensions).once.returns([60, 60]) + subject.calculate_dimensions(@upload, @file) do |w, h| + assert_operator(w, :>, 0) + assert_operator(h, :>, 0) + end + end + end + + context "for a video" do + setup do + @file = File.open("test/files/test-300x300.mp4", "rb") + @upload = mock() + @upload.stubs(:is_video?).returns(true) + end + + teardown do + @file.close + end + + should "return the dimensions" do + subject.calculate_dimensions(@upload, @file) do |w, h| + assert_operator(w, :>, 0) + assert_operator(h, :>, 0) + end + end + end + + context "for an image" do + setup do + @file = File.open("test/files/test.jpg", "rb") + @upload = mock() + @upload.stubs(:is_video?).returns(false) + @upload.stubs(:is_ugoira?).returns(false) + end + + teardown do + @file.close + end + + should "find the dimensions" do + subject.calculate_dimensions(@upload, @file) do |w, h| + assert_operator(w, :>, 0) + assert_operator(h, :>, 0) + end + end + end + end + + context ".process_file" do + setup do + @upload = FactoryBot.build(:jpg_upload) + @file = @upload.file + end + + should "run" do + subject.expects(:distribute_files).twice + subject.process_file(@upload, @file) + assert_equal("jpg", @upload.file_ext) + assert_equal(28086, @upload.file_size) + assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5) + assert_equal(335, @upload.image_height) + assert_equal(500, @upload.image_width) + end + end + + context ".generate_resizes" do + context "for an ugoira" do + setup do + context = UGOIRA_CONTEXT + @file = File.open("test/fixtures/ugoira.zip", "rb") + @upload = mock() + @upload.stubs(:is_video?).returns(false) + @upload.stubs(:is_ugoira?).returns(true) + @upload.stubs(:context).returns(context) + end + + should "generate a preview and a video" do + preview, sample = subject.generate_resizes(@file, @upload) + assert_operator(File.size(preview.path), :>, 0) + assert_operator(File.size(sample.path), :>, 0) + preview.close + preview.unlink + sample.close + sample.unlink + end + end + + context "for a video" do + teardown do + @file.close + end + + context "for an mp4" do + setup do + @file = File.open("test/files/test-300x300.mp4", "rb") + @upload = mock() + @upload.stubs(:is_video?).returns(true) + @upload.stubs(:is_ugoira?).returns(false) + end + + should "generate a video" do + preview, sample = subject.generate_resizes(@file, @upload) + assert_operator(File.size(preview.path), :>, 0) + preview.close + preview.unlink + end + end + + context "for a webm" do + setup do + @file = File.open("test/files/test-512x512.webm", "rb") + @upload = mock() + @upload.stubs(:is_video?).returns(true) + @upload.stubs(:is_ugoira?).returns(false) + end + + should "generate a video" do + preview, sample = subject.generate_resizes(@file, @upload) + assert_operator(File.size(preview.path), :>, 0) + preview.close + preview.unlink + end + end + end + + context "for an image" do + teardown do + @file.close + end + + setup do + @upload = mock() + @upload.stubs(:is_video?).returns(false) + @upload.stubs(:is_ugoira?).returns(false) + @upload.stubs(:is_image?).returns(true) + @upload.stubs(:image_width).returns(1200) + @upload.stubs(:image_height).returns(200) + end + + context "for a jpeg" do + setup do + @file = File.open("test/files/test.jpg", "rb") + end + + should "generate a preview" do + preview, sample = subject.generate_resizes(@file, @upload) + assert_operator(File.size(preview.path), :>, 0) + assert_operator(File.size(sample.path), :>, 0) + preview.close + preview.unlink + sample.close + sample.unlink + end + end + + context "for a png" do + setup do + @file = File.open("test/files/test.png", "rb") + end + + should "generate a preview" do + preview, sample = subject.generate_resizes(@file, @upload) + assert_operator(File.size(preview.path), :>, 0) + assert_operator(File.size(sample.path), :>, 0) + preview.close + preview.unlink + sample.close + sample.unlink + end + end + + context "for a gif" do + setup do + @file = File.open("test/files/test.png", "rb") + end + + should "generate a preview" do + preview, sample = subject.generate_resizes(@file, @upload) + assert_operator(File.size(preview.path), :>, 0) + assert_operator(File.size(sample.path), :>, 0) + preview.close + preview.unlink + sample.close + sample.unlink + end + end + end + end + + context ".generate_video_preview_for" do + context "for an mp4" do + setup do + @path = "test/files/test-300x300.mp4" + @video = FFMPEG::Movie.new(@path) + end + + should "generate a video" do + sample = subject.generate_video_preview_for(@video, 100, 100) + assert_operator(File.size(sample.path), :>, 0) + sample.close + sample.unlink + end + end + + context "for a webm" do + setup do + @path = "test/files/test-512x512.webm" + @video = FFMPEG::Movie.new(@path) + end + + should "generate a video" do + sample = subject.generate_video_preview_for(@video, 100, 100) + assert_operator(File.size(sample.path), :>, 0) + sample.close + sample.unlink + end + end + end + end + + context "::Preprocessor" do + subject { UploadService::Preprocessor } + + context "#download_from_source" do + setup do + @jpeg = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg" + @ugoira = "https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip" + end + + should "work on a jpeg" do + file = subject.new({}).download_from_source(@jpeg) do |context| + assert_not_nil(context[:downloaded_source]) + assert_not_nil(context[:source]) + end + + assert_operator(File.size(file.path), :>, 0) + file.close + end + + should "work on an ugoira url" do + file = subject.new({}).download_from_source(@ugoira, referer_url: "https://www.pixiv.net") do |context| + assert_not_nil(context[:downloaded_source]) + assert_not_nil(context[:source]) + assert_not_nil(context[:ugoira]) + end + + assert_operator(File.size(file.path), :>, 0) + file.close + end + end + + context "#start!" do + setup do + CurrentUser.user = travel_to(1.month.ago) do + FactoryBot.create(:user) + end + CurrentUser.ip_addr = "127.0.0.1" + @jpeg = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg" + @ugoira = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364" + @video = "https://www.sample-videos.com/video/mp4/720/big_buck_bunny_720p_1mb.mp4" + end + + teardown do + CurrentUser.user = nil + CurrentUser.ip_addr = nil + end + + should "work for a jpeg" do + @service = subject.new(source: @jpeg) + @upload = @service.start!(CurrentUser.id) + assert_equal("preprocessed", @upload.status) + assert_not_nil(@upload.md5) + assert_equal("jpg", @upload.file_ext) + assert_operator(@upload.file_size, :>, 0) + assert_not_nil(@upload.source) + assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :original))) + assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :large))) + assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :preview))) + end + + should "work for an ugoira" do + @service = subject.new(source: @ugoira) + @upload = @service.start!(CurrentUser.id) + assert_equal("preprocessed", @upload.status) + assert_not_nil(@upload.md5) + assert_equal("zip", @upload.file_ext) + assert_operator(@upload.file_size, :>, 0) + assert_not_nil(@upload.source) + assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "zip", :original))) + assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "zip", :large))) + end + + should "work for a video" do + @service = subject.new(source: @video) + @upload = @service.start!(CurrentUser.id) + assert_equal("preprocessed", @upload.status) + assert_not_nil(@upload.md5) + assert_equal("mp4", @upload.file_ext) + assert_operator(@upload.file_size, :>, 0) + assert_not_nil(@upload.source) + assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "mp4", :original))) + assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "mp4", :preview))) + end + + context "on timeout errors" do + setup do + HTTParty.stubs(:get).raises(Net::ReadTimeout) + end + + should "leave the upload in an error state" do + @service = subject.new(source: @video) + @upload = @service.start!(CurrentUser.id) + assert_match(/error:/, @upload.status) + end + end + + end + end + + context "::Replacer" do + context "for a file replacement" do + setup do + @new_file = upload_file("test/files/test.jpg") + @old_file = upload_file("test/files/test.png") + travel_to(1.month.ago) do + @user = FactoryBot.create(:user) + end + as_user do + @post = FactoryBot.create(:post, md5: Digest::MD5.hexdigest(@old_file.read)) + @old_md5 = @post.md5 + @post.stubs(:queue_delete_files) + @replacement = FactoryBot.create(:post_replacement, post: @post, replacement_url: "", replacement_file: @new_file) + end + end + + subject { UploadService::Replacer.new(post: @post, replacement: @replacement) } + + context "#process!" do + should "create a new upload" do + assert_difference(-> { Upload.count }) do + as_user { subject.process! } + end + end + + should "create a comment" do + assert_difference(-> { @post.comments.count }) do + as_user { subject.process! } + @post.reload + end + end + + should "not create a new post" do + assert_difference(-> { Post.count }, 0) do + as_user { subject.process! } + end + end + + should "update the post's MD5" do + assert_changes(-> { @post.md5 }) do + as_user { subject.process! } + @post.reload + end + end + + should "preserve the old values" do + as_user { subject.process! } + assert_equal(1500, @replacement.image_width_was) + assert_equal(1000, @replacement.image_height_was) + assert_equal(2000, @replacement.file_size_was) + assert_equal("jpg", @replacement.file_ext_was) + assert_equal(@old_md5, @replacement.md5_was) + end + + should "record the new values" do + as_user { subject.process! } + assert_equal(500, @replacement.image_width) + assert_equal(335, @replacement.image_height) + assert_equal(28086, @replacement.file_size) + assert_equal("jpg", @replacement.file_ext) + assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @replacement.md5) + end + + should "correctly update the attributes" do + as_user { subject.process! } + assert_equal(500, @post.image_width) + assert_equal(335, @post.image_height) + assert_equal(28086, @post.file_size) + assert_equal("jpg", @post.file_ext) + assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @post.md5) + assert(File.exists?(@post.file.path)) + end + end + + context "a post with the same file" do + should "not raise a duplicate error" do + upload_file("test/files/test.png") do |file| + assert_nothing_raised do + as_user { @post.replace!(replacement_file: file, replacement_url: "") } + end + end + end + + should "not queue a deletion or log a comment" do + upload_file("test/files/test.png") do |file| + assert_no_difference(-> { @post.comments.count }) do + as_user { @post.replace!(replacement_file: file, replacement_url: "") } + @post.reload + end + end + end + end + end + + context "for a source replacement" do + setup do + @new_url = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg" + travel_to(1.month.ago) do + @user = FactoryBot.create(:user) + end + as_user do + @post = FactoryBot.create(:post, uploader_ip_addr: "127.0.0.2") + @post.stubs(:queue_delete_files) + @replacement = FactoryBot.create(:post_replacement, post: @post, replacement_url: @new_url) + end + end + + subject { UploadService::Replacer.new(post: @post, replacement: @replacement) } + + context "a post when given a final_source" do + should "change the source to the final_source" do + replacement_url = "http://data.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_orwwptNBCE1wsfqepo1_raw.png" + final_source = "https://noizave.tumblr.com/post/162094447052" + + as_user { @post.replace!(replacement_url: replacement_url, final_source: final_source) } + + assert_equal(final_source, @post.source) + end + end + + context "a post when replaced with a HTML source" do + should "record the image URL as the replacement URL, not the HTML source" do + skip "Twitter key not set" unless Danbooru.config.twitter_api_key + replacement_url = "https://twitter.com/nounproject/status/540944400767922176" + image_url = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:orig" + as_user { @post.replace!(replacement_url: replacement_url) } + + assert_equal(image_url, @post.replacements.last.replacement_url) + end + end + + context "#undo!" do + setup do + @user = travel_to(1.month.ago) { FactoryBot.create(:user) } + as_user do + @post = FactoryBot.create(:post, source: "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png") + @post.stubs(:queue_delete_files) + @post.replace!(replacement_url: "https://danbooru.donmai.us/data/preview/download.png", tags: "-tag1 tag2") + end + + @replacement = @post.replacements.last + end + + should "update the attributes" do + as_user do + subject.undo! + end + + assert_equal("lowres tag2", @post.tag_string) + assert_equal(272, @post.image_width) + assert_equal(92, @post.image_height) + assert_equal(5969, @post.file_size) + assert_equal("png", @post.file_ext) + assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", @post.md5) + assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file).hexdigest) + assert_equal("https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png", @post.source) + end + end + + context "#process!" do + should "create a new upload" do + assert_difference(-> { Upload.count }) do + as_user { subject.process! } + end + end + + should "create a comment" do + assert_difference(-> { @post.comments.count }) do + as_user { subject.process! } + @post.reload + end + end + + should "not create a new post" do + assert_difference(-> { Post.count }, 0) do + as_user { subject.process! } + end + end + + should "update the post's MD5" do + assert_changes(-> { @post.md5 }) do + as_user { subject.process! } + @post.reload + end + end + + should "update the post's source" do + assert_changes(-> { @post.source }, nil, from: @post.source, to: @new_url) do + as_user { subject.process! } + @post.reload + end + end + + should "not change the post status or uploader" do + assert_no_changes(-> { {ip_addr: @post.uploader_ip_addr.to_s, uploader: @post.uploader_id, pending: @post.is_pending?} }) do + as_user { subject.process! } + @post.reload + end + end + + should "leave a system comment" do + as_user { subject.process! } + comment = @post.comments.last + assert_not_nil(comment) + assert_equal(User.system.id, comment.creator_id) + assert_match(/replaced this post/, comment.body) + end + end + + context "a post with a pixiv html source" do + setup do + Delayed::Worker.delay_jobs = true + end + + teardown do + Delayed::Worker.delay_jobs = false + end + + should "replace with the full size image" do + begin + as_user do + @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") + end + + assert_equal(80, @post.image_width) + assert_equal(82, @post.image_height) + assert_equal(16275, @post.file_size) + assert_equal("png", @post.file_ext) + assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5) + assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest) + assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url) + assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source) + rescue Net::OpenTimeout + skip "Remote connection to Pixiv failed" + end + end + + should "delete the old files after thirty days" do + begin + @post.unstub(:queue_delete_files) + FileUtils.expects(:rm_f).times(3) + + as_user { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") } + + travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do + Delayed::Worker.new.work_off + end + rescue Net::OpenTimeout + skip "Remote connection to Pixiv failed" + end + end + end + + context "a post that is replaced by a ugoira" do + should "save the frame data" do + skip "ffmpeg not installed" unless check_ffmpeg + begin + as_user { @post.replace!(replacement_url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") } + @post.reload + + assert_equal(80, @post.image_width) + assert_equal(82, @post.image_height) + assert_equal(2804, @post.file_size) + assert_equal("zip", @post.file_ext) + assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5) + assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest) + + assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source) + assert_equal([{"delay"=>125, "file"=>"000000.jpg"}, {"delay"=>125,"file"=>"000001.jpg"}], @post.pixiv_ugoira_frame_data.data) + rescue Net::OpenTimeout + skip "Remote connection to Pixiv failed" + end + end + end + + context "a post that is replaced to another file then replaced back to the original file" do + setup do + Delayed::Worker.delay_jobs = true + end + + teardown do + Delayed::Worker.delay_jobs = false + end + + should "not delete the original files" do + begin + FileUtils.expects(:rm_f).never + + as_user do + @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") + @post.reload + @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @post.reload + Upload.destroy_all + @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") + end + + assert_nothing_raised { @post.file(:original) } + assert_nothing_raised { @post.file(:preview) } + + travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do + Delayed::Worker.new.work_off + end + + assert_nothing_raised { @post.file(:original) } + assert_nothing_raised { @post.file(:preview) } + rescue Net::OpenTimeout + skip "Remote connection to Pixiv failed" + end + end + end + + context "two posts that have had their files swapped" do + setup do + Delayed::Worker.delay_jobs = true + + as_user do + @post1 = FactoryBot.create(:post) + @post2 = FactoryBot.create(:post) + end + end + + teardown do + Delayed::Worker.delay_jobs = false + end + + should "not delete the still active files" do + # swap the images between @post1 and @post2. + begin + as_user do + @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") + @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @post2.replace!(replacement_url: "https://www.google.com/intl/en_ALL/images/logo.gif") + Upload.destroy_all + @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") + end + + Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do + Delayed::Worker.new.work_off + end + + assert_nothing_raised { @post1.file(:original) } + assert_nothing_raised { @post2.file(:original) } + rescue Net::OpenTimeout + skip "Remote connection to Pixiv failed" + end + end + end + + context "a post with notes" do + setup do + Note.any_instance.stubs(:merge_version?).returns(false) + + as_user do + @post.update(image_width: 160, image_height: 164) + @note = @post.notes.create(x: 80, y: 82, width: 80, height: 82, body: "test") + @note.reload + end + end + + should "rescale the notes" do + assert_equal([80, 82, 80, 82], [@note.x, @note.y, @note.width, @note.height]) + + begin + assert_difference(-> { @note.versions.count }) do + # replacement image is 80x82, so we're downscaling by 50% (160x164 -> 80x82). + as_user do + @post.replace!(replacement_url: "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg") + end + @note.reload + end + + assert_equal([1024, 768, 1024, 768], [@note.x, @note.y, @note.width, @note.height]) + rescue Net::OpenTimeout + skip "Remote connection to Pixiv failed" + end + end + end + end + end + + context "#start!" do + subject { UploadService } + + setup do + @source = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg" + CurrentUser.user = travel_to(1.month.ago) do + FactoryBot.create(:user) + end + CurrentUser.ip_addr = "127.0.0.1" + end + + teardown do + CurrentUser.user = nil + CurrentUser.ip_addr = nil + end + + context "automatic tagging" do + setup do + @build_service = ->(file) { subject.new(file: file)} + end + + should "tag animated png files" do + service = @build_service.call(upload_file("test/files/apng/normal_apng.png")) + upload = service.start! + assert_match(/animated_png/, upload.tag_string) + end + + should "tag animated gif files" do + service = @build_service.call(upload_file("test/files/test-animated-86x52.gif")) + upload = service.start! + assert_match(/animated_gif/, upload.tag_string) + end + + should "not tag static gif files" do + service = @build_service.call(upload_file("test/files/test-static-32x32.gif")) + upload = service.start! + assert_no_match(/animated_gif/, upload.tag_string) + end + end + + context "that is too large" do + setup do + Danbooru.config.stubs(:max_image_resolution).returns(31*31) + end + + should "should fail validation" do + service = subject.new(file: upload_file("test/files/test-static-32x32.gif")) + upload = service.start! + assert_match(/image resolution is too large/, upload.status) + end + end + + context "with a preprocessing predecessor" do + setup do + @predecessor = FactoryBot.create(:source_upload, status: "preprocessing", source: @source, image_height: 0, image_width: 0, file_ext: "jpg") + Delayed::Worker.delay_jobs = true + end + + teardown do + Delayed::Worker.delay_jobs = false + end + + should "schedule a job later" do + service = subject.new(source: @source) + + assert_difference(-> { Delayed::Job.count }) do + predecessor = service.start! + assert_equal(@predecessor, predecessor) + end + end + end + + context "with a preprocessed predecessor" do + setup do + @predecessor = FactoryBot.create(:source_upload, status: "preprocessed", source: @source, file_size: 0, md5: "something", image_height: 0, image_width: 0, file_ext: "jpg") + @tags = 'hello world' + end + + should "update the predecessor" do + service = subject.new(source: @source, tag_string: @tags) + + predecessor = service.start! + assert_equal(@predecessor, predecessor) + assert_equal(@tags, predecessor.tag_string.strip) + end + end + + context "with no predecessor" do + should "create an upload" do + service = subject.new(source: @source) + + assert_difference(-> { Upload.count }) do + service.start! + end + end + end + end + + context "#create_post_from_upload" do + subject { UploadService } + + setup do + CurrentUser.user = travel_to(1.month.ago) do + FactoryBot.create(:user) + end + CurrentUser.ip_addr = "127.0.0.1" + end + + teardown do + CurrentUser.user = nil + CurrentUser.ip_addr = nil + end + + context "for an ugoira" do + setup do + @upload = FactoryBot.create(:ugoira_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, context: UGOIRA_CONTEXT) + end + + should "create a post" do + assert_difference(-> { PixivUgoiraFrameData.count }) do + post = subject.new({}).create_post_from_upload(@upload) + assert_equal([], post.errors.full_messages) + assert_not_nil(post.id) + end + end + end + + context "for an image" do + setup do + @upload = FactoryBot.create(:source_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100) + end + + should "create a commentary record" do + assert_difference(-> { ArtistCommentary.count }) do + subject.new({include_artist_commentary: true, artist_commentary_title: "blah", artist_commentary_desc: "blah"}).create_post_from_upload(@upload) + end + end + + should "create a post" do + post = subject.new({}).create_post_from_upload(@upload) + assert_equal([], post.errors.full_messages) + assert_not_nil(post.id) + end + end + + end +end diff --git a/test/unit/post_replacement_test.rb b/test/unit/post_replacement_test.rb index 7d96a07c4..5a20bbbf3 100644 --- a/test/unit/post_replacement_test.rb +++ b/test/unit/post_replacement_test.rb @@ -27,278 +27,11 @@ class PostReplacementTest < ActiveSupport::TestCase context "Replacing" do setup do CurrentUser.scoped(@uploader, "127.0.0.2") do - upload = FactoryBot.create(:jpg_upload, as_pending: "0", tag_string: "lowres tag1") - upload.process! + attributes = FactoryBot.attributes_for(:jpg_upload, as_pending: "0", tag_string: "lowres tag1") + service = UploadService.new(attributes) + upload = service.start! @post = upload.post end end - - context "a post from a generic source" do - setup do - @post.update(source: "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png") - @post.replace!(replacement_url: "https://www.google.com/intl/en_ALL/images/logo.gif", tags: "-tag1 tag2") - @replacement = @post.replacements.last - @upload = Upload.last - end - - context "that is then undone" do - setup do - Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do - Delayed::Worker.new.work_off - end - - @replacement = @post.replacements.first - @replacement.undo! - @post.reload - end - - should "update the attributes" do - assert_equal("lowres tag2", @post.tag_string) - assert_equal(272, @post.image_width) - assert_equal(92, @post.image_height) - assert_equal(5969, @post.file_size) - assert_equal("png", @post.file_ext) - assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", @post.md5) - assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file).hexdigest) - assert_equal("https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png", @post.source) - end - end - - should "create a post replacement record" do - assert_equal(@post.id, PostReplacement.last.post_id) - end - - should "record the old file metadata" do - assert_equal(500, @replacement.image_width_was) - assert_equal(335, @replacement.image_height_was) - assert_equal(28086, @replacement.file_size_was) - assert_equal("jpg", @replacement.file_ext_was) - assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @replacement.md5_was) - end - - should "record the new file metadata" do - assert_equal(276, @replacement.image_width) - assert_equal(110, @replacement.image_height) - assert_equal(8558, @replacement.file_size) - assert_equal("gif", @replacement.file_ext) - assert_equal("e80d1c59a673f560785784fb1ac10959", @replacement.md5) - end - - should "correctly update the attributes" do - assert_equal(@post.id, @upload.post.id) - assert_equal("completed", @upload.status) - - assert_equal(276, @post.image_width) - assert_equal(110, @post.image_height) - assert_equal(8558, @post.file_size) - assert_equal("gif", @post.file_ext) - assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5) - assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file).hexdigest) - assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source) - end - - should "not change the post status or uploader" do - assert_equal("127.0.0.2", @post.uploader_ip_addr.to_s) - assert_equal(@uploader.id, @post.uploader_id) - assert_equal(false, @post.is_pending) - end - - should "leave a system comment" do - comment = @post.comments.last - - assert_not_nil(comment) - assert_equal(User.system.id, comment.creator_id) - assert_match(/replaced this post/, comment.body) - end - - should "not send an @mention to the replacer" do - assert_equal(0, @replacer.dmails.size) - end - end - - context "a post with notes" do - setup do - @post.update(image_width: 160, image_height: 164) - CurrentUser.scoped(@uploader, "127.0.0.1") do - @note = @post.notes.create(x: 80, y: 82, width: 80, height: 82, body: "test") - end - end - - should "rescale the notes" do - assert_equal([80, 82, 80, 82], [@note.x, @note.y, @note.width, @note.height]) - - begin - assert_difference("@replacer.note_versions.count") do - # replacement image is 80x82, so we're downscaling by 50% (160x164 -> 80x82). - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - @note.reload - end - - assert_equal([40, 41, 40, 41], [@note.x, @note.y, @note.width, @note.height]) - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" - end - end - end - - context "a post with a pixiv html source" do - should "replace with the full size image" do - begin - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - - assert_equal(80, @post.image_width) - assert_equal(82, @post.image_height) - assert_equal(16275, @post.file_size) - assert_equal("png", @post.file_ext) - assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5) - assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest) - assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source) - assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url) - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" - end - end - - should "delete the old files after thirty days" do - begin - old_file_path, old_preview_file_path = @post.file(:original).path, @post.file(:preview).path - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - - assert(File.exists?(old_file_path)) - assert(File.exists?(old_preview_file_path)) - - Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do - Delayed::Worker.new.work_off - end - - assert_not(File.exists?(old_file_path)) - assert_not(File.exists?(old_preview_file_path)) - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" - end - end - end - - context "a post that is replaced by a ugoira" do - should "save the frame data" do - skip "ffmpeg not installed" unless check_ffmpeg - begin - @post.replace!(replacement_url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - @post.reload - - assert_equal(80, @post.image_width) - assert_equal(82, @post.image_height) - assert_equal(2804, @post.file_size) - assert_equal("zip", @post.file_ext) - assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5) - assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest) - - assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source) - assert_equal([{"delay"=>125, "file"=>"000001.jpg"}, {"delay"=>125,"file"=>"000002.jpg"}], @post.pixiv_ugoira_frame_data.data) - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" - end - end - end - - context "a post that is replaced to another file then replaced back to the original file" do - should "not delete the original files" do - skip "ffmpeg is not installed" unless check_ffmpeg - - begin - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - - assert_nothing_raised { @post.file(:original) } - assert_nothing_raised { @post.file(:preview) } - - Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do - Delayed::Worker.new.work_off - end - - assert_nothing_raised { @post.file(:original) } - assert_nothing_raised { @post.file(:preview) } - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" - end - end - end - - context "two posts that have had their files swapped" do - should "not delete the still active files" do - skip "ffmpeg is not installed" unless check_ffmpeg - - @post1 = FactoryBot.create(:post) - @post2 = FactoryBot.create(:post) - - # swap the images between @post1 and @post2. - begin - @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - @post2.replace!(replacement_url: "https://www.google.com/intl/en_ALL/images/logo.gif") - @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - - Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do - Delayed::Worker.new.work_off - end - - assert_nothing_raised { @post1.file(:original) } - assert_nothing_raised { @post2.file(:original) } - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" - end - end - end - - context "a post with an uploaded file" do - should "work" do - upload_file("test/files/test.png") do |file| - @post.replace!(replacement_file: file, replacement_url: "") - assert_equal(@post.md5, Digest::MD5.file(file.tempfile).hexdigest) - assert_equal("file://test.png", @post.replacements.last.replacement_url) - end - end - end - - context "a post when given a final_source" do - should "change the source to the final_source" do - replacement_url = "http://data.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_orwwptNBCE1wsfqepo1_raw.png" - final_source = "https://noizave.tumblr.com/post/162094447052" - @post.replace!(replacement_url: replacement_url, final_source: final_source) - - assert_equal(final_source, @post.source) - end - end - - context "a post when replaced with a HTML source" do - should "record the image URL as the replacement URL, not the HTML source" do - skip "Twitter key not set" unless Danbooru.config.twitter_api_key - replacement_url = "https://twitter.com/nounproject/status/540944400767922176" - image_url = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:orig" - @post.replace!(replacement_url: replacement_url) - - assert_equal(image_url, @post.replacements.last.replacement_url) - end - end - - context "a post with the same file" do - should "not raise a duplicate error" do - upload_file("test/files/test.jpg") do |file| - assert_nothing_raised do - @post.replace!(replacement_file: file, replacement_url: "") - end - end - end - - should "not queue a deletion or log a comment" do - upload_file("test/files/test.jpg") do |file| - assert_no_difference(["@post.comments.count"]) do - @post.replace!(replacement_file: file, replacement_url: "") - end - end - end - end end end diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index 30dcb0110..f894f254e 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -28,8 +28,7 @@ class PostTest < ActiveSupport::TestCase context "Deletion:" do context "Expunging a post" do setup do - @upload = FactoryBot.create(:jpg_upload) - @upload.process! + @upload = UploadService.new(FactoryBot.attributes_for(:jpg_upload)).start! @post = @upload.post Favorite.add(post: @post, user: @user) end @@ -2677,4 +2676,19 @@ class PostTest < ActiveSupport::TestCase end end end + + context "#replace!" do + subject { @post.replace!(tags: "something", replacement_url: "https://danbooru.donmai.us/data/preview/download.png") } + + setup do + @post = FactoryBot.create(:post) + @post.stubs(:queue_delete_files) + end + + should "update the post" do + assert_changes(-> { @post.md5 }) do + subject + end + end + end end diff --git a/test/unit/upload_test.rb b/test/unit/upload_test.rb index ef81946a5..b5cad216e 100644 --- a/test/unit/upload_test.rb +++ b/test/unit/upload_test.rb @@ -1,6 +1,8 @@ require 'test_helper' class UploadTest < ActiveSupport::TestCase + SOURCE_URL = "https://upload.wikimedia.org/wikipedia/commons/thumb/6/66/NAMA_Machine_d%27Anticyth%C3%A8re_1.jpg/538px-NAMA_Machine_d%27Anticyth%C3%A8re_1.jpg?download" + context "In all cases" do setup do mock_iqdb_service! @@ -28,298 +30,12 @@ class UploadTest < ActiveSupport::TestCase end end - context "image size calculator" do - should "discover the dimensions for a compressed SWF" do - @upload = FactoryBot.create(:upload, file: upload_file("test/files/compressed.swf")) - assert_equal([607, 756], @upload.calculate_dimensions) - end - - should "discover the dimensions for a JPG with JFIF data" do - @upload = FactoryBot.create(:jpg_upload) - assert_equal([500, 335], @upload.calculate_dimensions) - end - - should "discover the dimensions for a JPG with EXIF data" do - @upload = FactoryBot.create(:upload, file: upload_file("test/files/test-exif-small.jpg")) - assert_equal([529, 600], @upload.calculate_dimensions) - end - - should "discover the dimensions for a JPG with no header data" do - @upload = FactoryBot.create(:upload, file: upload_file("test/files/test-blank.jpg")) - assert_equal([668, 996], @upload.calculate_dimensions) - end - - should "discover the dimensions for a PNG" do - @upload = FactoryBot.create(:upload, file: upload_file("test/files/test.png")) - assert_equal([768, 1024], @upload.calculate_dimensions) - end - - should "discover the dimensions for a GIF" do - @upload = FactoryBot.create(:upload, file: upload_file("test/files/test.gif")) - assert_equal([400, 400], @upload.calculate_dimensions) - end - end - - context "content type calculator" do - should "know how to parse jpeg, png, gif, and swf file headers" do - @upload = FactoryBot.create(:jpg_upload) - assert_equal("jpg", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.jpg"))) - assert_equal("gif", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.gif"))) - assert_equal("png", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.png"))) - assert_equal("swf", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/compressed.swf"))) - assert_equal("bin", @upload.file_header_to_file_ext(File.open("#{Rails.root}/README.md"))) - end - end - - context "downloader" do - context "for a zip that is not an ugoira" do - should "not validate" do - @upload = FactoryBot.create(:upload, file: upload_file("test/files/invalid_ugoira.zip")) - @upload.process! - assert_equal("error: RuntimeError - missing frame data for ugoira", @upload.status) - end - end - - context "that is a pixiv ugoira" do - setup do - @url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654" - @upload = FactoryBot.create(:source_upload, :source => @url, :tag_string => "ugoira") - end - - should "process successfully" do - begin - _, _, output_file = @upload.download_from_source(@url, "") - assert_operator(output_file.size, :>, 1_000) - assert_equal("zip", @upload.file_header_to_file_ext(output_file)) - rescue Net::OpenTimeout - skip "Remote connection to #{@url} failed" - end - end - end - end - - context "determining if a file is downloadable" do - should "classify HTTP sources as downloadable" do - @upload = FactoryBot.create(:source_upload, :source => "http://www.google.com/1.jpg") - assert_not_nil(@upload.is_downloadable?) - end - - should "classify HTTPS sources as downloadable" do - @upload = FactoryBot.create(:source_upload, :source => "https://www.google.com/1.jpg") - assert_not_nil(@upload.is_downloadable?) - end - - should "classify non-HTTP/HTTPS sources as not downloadable" do - @upload = FactoryBot.create(:source_upload, :source => "ftp://www.google.com/1.jpg") - assert_nil(@upload.is_downloadable?) - end - end - - context "file processor" do - should "parse and process a cgi file representation" do - @upload = FactoryBot.create(:upload, file: upload_file("test/files/test.jpg")) - assert_nothing_raised {@upload.process_upload} - assert_equal(28086, @upload.file_size) - end - - should "process a transparent png" do - @upload = FactoryBot.create(:upload, file: upload_file("test/files/alpha.png")) - assert_nothing_raised {@upload.process_upload} - assert_equal(1136, @upload.file_size) - end - end - - context "hash calculator" do - should "caculate the hash" do - @upload = FactoryBot.create(:jpg_upload) - @upload.process_upload - assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5) - end - end - - context "resizer" do - should "generate several resized versions of the image" do - @upload = FactoryBot.create(:upload, file_ext: "jpg", image_width: 1356, image_height: 911, file: upload_file("test/files/test-large.jpg")) - preview_file, sample_file = @upload.generate_resizes - assert_operator(preview_file.size, :>, 1_000) - assert_operator(sample_file.size, :>, 1_000) - end - end - should "increment the uploaders post_upload_count" do - @upload = FactoryBot.create(:source_upload) - assert_difference("CurrentUser.user.post_upload_count", 1) do - @upload.process! + assert_difference(-> { CurrentUser.user.post_upload_count }) do + FactoryBot.create(:source_upload) CurrentUser.user.reload end end - - context "with an artist commentary" do - setup do - @upload = FactoryBot.create(:source_upload, - include_artist_commentary: "1", - artist_commentary_title: "", - artist_commentary_desc: "blah", - ) - end - - should "create an artist commentary when processed" do - assert_difference("ArtistCommentary.count") do - @upload.process! - end - end - end - - should "process completely for a downloaded image" do - @upload = FactoryBot.create(:source_upload, - :rating => "s", - :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo" - ) - assert_difference("Post.count") do - assert_nothing_raised {@upload.process!} - end - - post = Post.last - assert_equal("http://www.google.com/intl/en_ALL/images/logo.gif", post.source) - assert_equal("foo hoge lowres", post.tag_string) - assert_equal("s", post.rating) - assert_equal(@upload.uploader_id, post.uploader_id) - assert_equal("127.0.0.1", post.uploader_ip_addr.to_s) - assert_equal(@upload.md5, post.md5) - assert_equal("gif", post.file_ext) - assert_equal(276, post.image_width) - assert_equal(110, post.image_height) - assert_equal(8558, post.file_size) - assert_equal(post.id, @upload.post_id) - assert_equal("completed", @upload.status) - end - - context "automatic tagging" do - should "tag animated png files" do - @upload = FactoryBot.build(:upload, file_ext: "png", file: upload_file("test/files/apng/normal_apng.png")) - assert_equal("animated_png", @upload.automatic_tags) - end - - should "tag animated gif files" do - @upload = FactoryBot.build(:upload, file_ext: "gif", file: upload_file("test/files/test-animated-86x52.gif")) - assert_equal("animated_gif", @upload.automatic_tags) - end - - should "not tag static gif files" do - @upload = FactoryBot.build(:upload, file_ext: "gif", file: upload_file("test/files/test-static-32x32.gif")) - assert_equal("", @upload.automatic_tags) - end - end - - context "that is too large" do - should "should fail validation" do - Danbooru.config.stubs(:max_image_resolution).returns(31*31) - @upload = FactoryBot.create(:upload, file: upload_file("test/files/test-static-32x32.gif")) - @upload.process! - assert_match(/image resolution is too large/, @upload.status) - end - end - end - - should "process completely for a pixiv ugoira" do - skip "ffmpeg is not installed" unless check_ffmpeg - @upload = FactoryBot.create(:source_upload, source: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654") - - assert_difference(["PixivUgoiraFrameData.count", "Post.count"]) do - @upload.process! - assert_equal([], @upload.errors.full_messages) - end - post = Post.last - assert_not_nil(post.pixiv_ugoira_frame_data) - assert_equal("0d94800c4b520bf3d8adda08f95d31e2", post.md5) - assert_equal(60, post.image_width) - assert_equal(60, post.image_height) - assert_equal("https://i.pximg.net/img-zip-ugoira/img/2014/10/05/23/42/23/46378654_ugoira1920x1080.zip", post.source) - assert_nothing_raised { post.file(:original) } - assert_nothing_raised { post.file(:preview) } - end - - should "process completely for an uploaded image" do - @upload = FactoryBot.create(:jpg_upload, - :rating => "s", - :uploader_ip_addr => "127.0.0.1", - :tag_string => "hoge foo", - :file => upload_file("test/files/test.jpg"), - ) - - assert_difference("Post.count") do - assert_nothing_raised {@upload.process!} - end - post = Post.last - assert_equal("foo hoge lowres", post.tag_string) - assert_equal("s", post.rating) - assert_equal(@upload.uploader_id, post.uploader_id) - assert_equal("127.0.0.1", post.uploader_ip_addr.to_s) - assert_equal(@upload.md5, post.md5) - assert_equal("jpg", post.file_ext) - assert_nothing_raised { post.file(:original) } - assert_equal(28086, post.file(:original).size) - assert_equal(post.id, @upload.post_id) - assert_equal("completed", @upload.status) - end - - should "process completely for a .webm" do - upload = FactoryBot.create(:upload, rating: "s", file: upload_file("test/files/test-512x512.webm")) - - assert_difference("Post.count") do - upload.process! - assert_equal("completed", upload.status) - end - - post = Post.last - assert_includes(post.tag_array, "webm") - assert_equal("webm", upload.file_ext) - assert_equal(12345, upload.file_size) - assert_equal(512, upload.image_width) - assert_equal(512, upload.image_height) - assert_equal("34dd2489f7aaa9e57eda1b996ff26ff7", upload.md5) - - assert_nothing_raised { post.file(:preview) } - assert_nothing_raised { post.file(:original) } - end - - should "process completely for a .mp4" do - upload = FactoryBot.create(:upload, rating: "s", file: upload_file("test/files/test-300x300.mp4")) - - assert_difference("Post.count") do - upload.process! - assert_equal("completed", upload.status) - end - - post = Post.last - assert_includes(post.tag_array, "mp4") - assert_equal("mp4", upload.file_ext) - assert_equal(18677, upload.file_size) - assert_equal(300, upload.image_width) - assert_equal(300, upload.image_height) - assert_equal("865c93102cad3e8a893d6aac6b51b0d2", upload.md5) - - assert_nothing_raised { post.file(:preview) } - assert_nothing_raised { post.file(:original) } - end - - should "process completely for a null source" do - @upload = FactoryBot.create(:jpg_upload, :source => nil) - - assert_difference("Post.count") do - assert_nothing_raised {@upload.process!} - end - end - - context "on timeout errors" do - should "leave the upload in an error state" do - HTTParty.stubs(:get).raises(Net::ReadTimeout) - @upload = FactoryBot.create(:source_upload) - @upload.process! - - assert_match(/\Aerror/, @upload.status) - end end end end