add support for upload preprocessing

Procfile

@@ -1,2 +1,2 @@
 unicorn: bundle exec rails server
-jobs: bundle exec script/delayed_job run
+jobs: bundle exec script/delayed_job run 

app/controllers/uploads_controller.rb

@@ -3,31 +3,14 @@ class UploadsController < ApplicationController
   respond_to :html, :xml, :json, :js
 
   def new
-    @upload = Upload.new
     @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
-    if params[:url]
+    @upload, @post, @source, @normalized_url, @remote_size = UploadService::ControllerHelper.prepare(params[:url], params[:ref])
-      download = Downloads::File.new(params[:url])
-      @normalized_url, _, _ = download.before_download(params[:url], {})
-      @post = find_post_by_url(@normalized_url)
-
-      begin
-        @source = Sources::Site.new(params[:url], :referer_url => params[:ref])
-        @remote_size = download.size
-      rescue Exception
-      end
-    end
     respond_with(@upload)
   end
 
   def batch
     @url = params.dig(:batch, :url) || params[:url]
-    @source = nil
+    @source = UploadService::ControllerHelper.batch(@url, params[:ref])
-
-    if @url
-      @source = Sources::Site.new(@url, :referer_url => params[:ref])
-      @source.get
-    end
-
     respond_with(@source)
   end
 
@@ -57,14 +40,11 @@ class UploadsController < ApplicationController
   end
 
   def create
-    @upload = Upload.create(upload_params)
+    @service = UploadService.new(upload_params)
+    @upload = @service.start!
 
-    if @upload.errors.empty?
+    if @service.warnings.any?
-      post = @upload.process!
+      flash[:notice] = @service.warnings.join(".\n \n")
-
-      if post.present? && post.valid? && post.warnings.any?
-        flash[:notice] = post.warnings.full_messages.join(".\n \n")
-      end
     end
 
     save_recent_tags
@@ -73,14 +53,6 @@ class UploadsController < ApplicationController
 
   private
 
-  def find_post_by_url(normalized_url)
-    if normalized_url.nil?
-      Post.where("SourcePattern(lower(posts.source)) = ?", params[:url]).first
-    else
-      Post.where("SourcePattern(lower(posts.source)) IN (?)", [params[:url], @normalized_url]).first
-    end
-  end
-
   def save_recent_tags
     if @upload
       tags = Tag.scan_tags(@upload.tag_string)
@@ -94,7 +66,7 @@ class UploadsController < ApplicationController
     permitted_params = %i[
       file source tag_string rating status parent_id artist_commentary_title
       artist_commentary_desc include_artist_commentary referer_url
-      md5_confirmation as_pending 
+      md5_confirmation as_pending
     ]
 
     params.require(:upload).permit(permitted_params)

app/logical/pixiv_ugoira_service.rb

@@ -1,33 +0,0 @@
-class PixivUgoiraService
-  attr_reader :width, :height, :frame_data, :content_type
-
-  def save_frame_data(post)
-    PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id)
-  end
-
-  def calculate_dimensions(source_path)
-    folder = Zip::File.new(source_path)
-    tempfile = Tempfile.new("ugoira-dimensions")
-
-    begin
-      folder.first.extract(tempfile.path) {true}
-      image_size = ImageSpec.new(tempfile.path)
-      @width = image_size.width
-      @height = image_size.height
-    ensure
-      tempfile.close
-      tempfile.unlink
-    end
-  end
-
-  def load(data)
-    if data[:is_ugoira]
-      @frame_data = data[:ugoira_frame_data]
-      @content_type = data[:ugoira_content_type]
-    end
-  end
-
-  def empty?
-    @frame_data.nil?
-  end
-end

app/logical/upload_service.rb

@@ -0,0 +1,399 @@
+class UploadService
+  module ControllerHelper
+    def self.prepare(url, ref = nil)
+      upload = Upload.new
+
+      if url
+        Preprocessor.new(source: url).delay(queue: "default").start!(CurrentUser.user.id)
+
+        download = Downloads::File.new(url)
+        normalized_url, _, _ = download.before_download(url, {})
+        post = if normalized_url.nil?
+          Post.where("SourcePattern(lower(posts.source)) = ?", url).first
+        else
+          Post.where("SourcePattern(lower(posts.source)) IN (?)", [url, normalized_url]).first
+        end
+
+        begin
+          source = Sources::Site.new(url, :referer_url => ref)
+          remote_size = download.size
+        rescue Exception
+        end
+
+        return [upload, post, source, normalized_url, remote_size]
+      end
+
+      return [upload]
+    end
+
+    def self.batch(url, ref = nil)
+      if url
+        source = Sources::Site.new(url, :referer_url => ref)
+        source.get
+        return source
+      end
+    end
+  end
+
+  module Utils
+    def self.file_header_to_file_ext(file)
+      case File.read(file.path, 16)
+      when /^\xff\xd8/n
+        "jpg"
+      when /^GIF87a/, /^GIF89a/
+        "gif"
+      when /^\x89PNG\r\n\x1a\n/n
+        "png"
+      when /^CWS/, /^FWS/, /^ZWS/
+        "swf"
+      when /^\x1a\x45\xdf\xa3/n
+        "webm"
+      when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
+        "mp4"
+      when /^PK\x03\x04/
+        "zip"
+      else
+        "bin"
+      end
+    end
+
+    def self.calculate_ugoira_dimensions(source_path)
+      folder = Zip::File.new(source_path)
+      Tempfile.open("ugoira-dim-") do |tempfile|
+        folder.first.extract(tempfile.path) { true }
+        image_size = ImageSpec.new(tempfile.path)
+        return [image_size.width, image_size.height]
+      end
+    end
+
+    def self.calculate_dimensions(upload, file)
+      if upload.is_video?
+        video = FFMPEG::Movie.new(file.path)
+        yield(video.width, video.height)
+
+      elsif upload.is_ugoira?
+        w, h = calculate_ugoira_dimensions(file.path)
+        yield(w, h)
+
+      else
+        image_size = ImageSpec.new(file.path)
+        yield(image_size.width, image_size.height)
+      end
+    end
+
+    def self.distribute_files(file, record, type)
+      [Danbooru.config.storage_manager, Danbooru.config.backup_storage_manager].each do |sm|
+        sm.store_file(file, record, type)
+      end
+    end
+
+    def self.is_downloadable?(source)
+      source.match?(/^https?:\/\//)
+    end
+
+    def self.generate_resizes(file, upload)
+      if upload.is_video?
+        video = FFMPEG::Movie.new(file.path)
+        preview_file = generate_video_preview_for(video, Danbooru.config.small_image_width, Danbooru.config.small_image_width)
+
+      elsif upload.is_ugoira?
+        preview_file = PixivUgoiraConverter.generate_preview(file)
+        sample_file = PixivUgoiraConverter.generate_webm(file, upload.context["ugoira"]["frame_data"])
+
+      elsif upload.is_image?
+        preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
+
+        if upload.image_width > Danbooru.config.large_image_width
+          sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, upload.image_height, 90)
+        end
+      end
+
+      [preview_file, sample_file]
+    end
+
+    def self.generate_video_preview_for(video, width, height)
+      dimension_ratio = video.width.to_f / video.height
+      if dimension_ratio > 1
+        height = (width / dimension_ratio).to_i
+      else
+        width = (height * dimension_ratio).to_i
+      end
+
+      output_file = Tempfile.new(binmode: true)
+      video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
+      output_file
+    end
+
+    def self.process_file(upload, file)
+      upload.file = file
+      upload.file_ext = Utils.file_header_to_file_ext(file)
+      upload.file_size = file.size
+      upload.md5 = Digest::MD5.file(file.path).hexdigest
+
+      Utils.calculate_dimensions(upload, file) do |width, height|
+        upload.image_width = width
+        upload.image_height = height
+      end
+
+      upload.tag_string = "#{upload.tag_string} #{Utils.automatic_tags(upload, file)}"
+
+      preview_file, sample_file = Utils.generate_resizes(file, upload)
+
+      begin
+        Utils.distribute_files(file, upload, :original)
+        Utils.distribute_files(sample_file, upload, :large) if sample_file.present?
+        Utils.distribute_files(preview_file, upload, :preview) if preview_file.present?
+      ensure
+        preview_file.try(:close!)
+        sample_file.try(:close!)
+      end
+    end
+
+    # these methods are only really used during upload processing even 
+    # though logically they belong on upload. post can rely on the 
+    # automatic tag that's added.
+    def self.is_animated_gif?(upload, file)
+      return false if upload.file_ext != "gif"
+
+      # Check whether the gif has multiple frames by trying to load the second frame.
+      result = Vips::Image.gifload(file.path, page: 1) rescue $ERROR_INFO
+      if result.is_a?(Vips::Image)
+        true
+      elsif result.is_a?(Vips::Error) && result.message =~ /too few frames in GIF file/
+        false
+      else
+        raise result
+      end
+    end
+
+    def self.is_animated_png?(upload, file)
+      upload.file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
+    end
+
+    def self.is_video_with_audio?(upload, file)
+      video = FFMPEG::Movie.new(file.path)
+      upload.is_video? && video.audio_channels.present?
+    end
+
+    def self.automatic_tags(upload, file)
+      return "" unless Danbooru.config.enable_dimension_autotagging
+
+      tags = []
+      tags << "video_with_sound" if is_video_with_audio?(upload, file)
+      tags << "animated_gif" if is_animated_gif?(upload, file)
+      tags << "animated_png" if is_animated_png?(upload, file)
+      tags.join(" ")
+    end
+  end
+
+  class Preprocessor
+    attr_reader :params
+
+    def initialize(params)
+      @params = params
+    end
+
+    def source
+      params[:source]
+    end
+
+    def in_progress?
+      Upload.where(status: "preprocessing", source: source).exists?
+    end
+
+    def predecessor
+      Upload.where(status: ["preprocessed", "preprocessing"], source: source).first
+    end
+
+    def completed?
+      predecessor.present?
+    end
+
+    def start!(uploader_id)
+      if !Utils.is_downloadable?(source)
+        return
+      end
+
+      if Post.where(source: source).exists?
+        return
+      end
+
+      if Upload.where(source: source, status: "completed").exists?
+        return
+      end
+
+      if Upload.where(source: source).where("status like ?", "error%").exists?
+        return
+      end
+
+      params[:rating] ||= "q"
+      params[:tag_string] ||= "tagme"
+
+      CurrentUser.as(User.find(uploader_id)) do
+        upload = Upload.create!(params)
+
+        upload.update(status: "preprocessing")
+
+        begin
+          file = download_from_source(source, referer_url: upload.referer_url) do |context|
+            upload.downloaded_source = context[:downloaded_source]
+            upload.source = context[:source]
+
+            if context[:ugoira]
+              upload.context = { ugoira: context[:ugoira] }
+            end
+          end
+
+          Utils.process_file(upload, file)
+
+          upload.rating = params[:rating]
+          upload.tag_string = params[:tag_string]
+          upload.status = "preprocessed"
+          upload.save!
+        rescue Exception => x
+          upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
+        end
+
+        return upload
+      end
+    end
+
+    def finish!
+      pred = self.predecessor()
+      pred.attributes = self.params
+      pred.status = "completed"
+      pred.save
+      return pred
+    end
+
+    def download_from_source(source, referer_url: nil)
+      download = Downloads::File.new(source, referer_url: referer_url)
+      file = download.download!
+      context = {
+        downloaded_source: download.downloaded_source,
+        source: download.source
+      }
+
+      if download.data[:is_ugoira]
+        context[:ugoira] = {
+          frame_data: download.data[:ugoira_frame_data],
+          content_type: download.data[:ugoira_content_type]
+        }
+      end
+
+      yield(context)
+
+      return file
+    end
+  end
+
+  attr_reader :params, :post, :upload
+
+  def initialize(params)
+    @params = params
+  end
+
+  def start!
+    preprocessor = Preprocessor.new(params)
+
+    if preprocessor.in_progress?
+      delay(queue: "default", run_at: 5.seconds.from_now).start!
+      return preprocessor.predecessor
+    end
+
+    if preprocessor.completed?
+      @upload = preprocessor.finish!
+      create_post_from_upload(@upload)
+      return @upload
+    end
+
+    params[:rating] ||= "q"
+    params[:tag_string] ||= "tagme"
+    @upload = Upload.create!(params)
+
+    begin
+      if @upload.invalid?
+        return @upload
+      end
+
+      @upload.update(status: "processing")
+
+      if @upload.file.present?
+        Utils.process_file(upload, @upload.file)
+      else
+        # sources will be handled in preprocessing now
+      end
+
+      @upload.save!
+      @post = create_post_from_upload(@upload)
+      return @upload
+
+    rescue Exception => x
+      @upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
+      @upload
+    end
+  end
+
+  def warnings
+    return [] if @post.nil?
+    return @post.warnings.full_messages
+  end
+
+  def source
+    params[:source]
+  end
+
+  def include_artist_commentary?
+    params[:include_artist_commentary].to_s.truthy?
+  end
+
+  def create_post_from_upload(upload)
+    @post = convert_to_post(upload)
+    @post.save!
+
+    @upload.update(status: "error: " + @post.errors.full_messages.join(", "))
+
+    if upload.context && upload.context["ugoira"]
+      PixivUgoiraFrameData.create(
+        post_id: @post.id,
+        data: upload.context["ugoira"]["frame_data"],
+        content_type: upload.context["ugoira"]["content_type"]
+      )
+    end
+
+    if include_artist_commentary?
+      @post.create_artist_commentary(
+        :original_title => params[:artist_commentary_title],
+        :original_description => params[:artist_commentary_desc]
+      )
+    end
+
+    notify_cropper(@post) if ImageCropper.enabled?
+    upload.update(status: "completed", post_id: @post.id)
+    @post
+  end
+
+  def convert_to_post(upload)
+    Post.new.tap do |p|
+      p.tag_string = upload.tag_string
+      p.md5 = upload.md5
+      p.file_ext = upload.file_ext
+      p.image_width = upload.image_width
+      p.image_height = upload.image_height
+      p.rating = upload.rating
+      p.source = upload.source
+      p.file_size = upload.file_size
+      p.uploader_id = upload.uploader_id
+      p.uploader_ip_addr = upload.uploader_ip_addr
+      p.parent_id = upload.parent_id
+
+      if !upload.uploader.can_upload_free? || upload.upload_as_pending?
+        p.is_pending = true
+      end
+    end
+  end
+
+  def notify_cropper(post)
+    # ImageCropper.notify(post)
+  end
+end

app/models/upload.rb

@@ -3,18 +3,67 @@ require "tmpdir"
 class Upload < ApplicationRecord
   class Error < Exception ; end
 
-  attr_accessor :file, :image_width, :image_height, :file_ext, :md5, 
+  class Validator < ActiveModel::Validator
-    :file_size, :as_pending, :artist_commentary_title, 
+    def validate(record)
-    :artist_commentary_desc, :include_artist_commentary,
+      if record.new_record?
-    :referer_url, :downloaded_source, :replaced_post
+        validate_md5_uniqueness(record)
-  belongs_to :uploader, :class_name => "User"
+        validate_video_duration(record)
+      end
 
+      validate_resolution(record)
+    end
+
+    def validate_md5_uniqueness(record)
+      if record.md5.nil?
+        return
+      end
+
+      md5_post = Post.find_by_md5(record.md5)
+
+      if md5_post.nil?
+        return
+      end
+
+      if record.replaced_post && record.replaced_post == md5_post
+        return
+      end
+
+      record.errors[:md5] << "duplicate: #{md5_post.id}"
+    end
+
+    def validate_resolution(record)
+      resolution = record.image_width.to_i * record.image_height.to_i
+
+      if resolution > Danbooru.config.max_image_resolution
+        record.errors[:base] << "image resolution is too large (resolution: #{(resolution / 1_000_000.0).round(1)} megapixels (#{record.image_width}x#{record.image_height}); max: #{Danbooru.config.max_image_resolution / 1_000_000} megapixels)"
+      end
+    end
+
+    def validate_video_duration(record)
+      if record.is_video? && record.video.duration > 120
+        record.errors[:base] << "video must not be longer than 2 minutes"
+      end
+    end
+  end
+
+
+  attr_accessor :as_pending,
+    :referer_url, :downloaded_source, :replaced_post, :file
+  belongs_to :uploader, :class_name => "User"
   belongs_to :post, optional: true
 
   before_validation :initialize_attributes
-  validate :uploader_is_not_limited, :on => :create
+  before_validation :assign_rating_from_tags
-  validate :file_or_source_is_present, :on => :create
+  validate :uploader_is_not_limited, on: :create
-  validate :rating_given
+  # validates :source, format: { with: /\Ahttps?/ }, if: ->(record) {record.file.blank?}, on: :create
+  validates :image_height, numericality: { less_than_or_equal_to: Danbooru.config.max_image_height }, allow_nil: true
+  validates :image_width, numericality: { less_than_or_equal_to: Danbooru.config.max_image_width }, allow_nil: true
+  validates :rating, inclusion: { in: %w(q e s) }, allow_nil: true
+  validates :md5, confirmation: true
+  validates :file_ext, format: { with: /jpg|gif|png|swf|webm|mp4|zip/ }, allow_nil: true
+  validates_with Validator
+  serialize :context, JSON
+  after_create {|rec| rec.uploader.increment!(:post_upload_count)}
 
   def initialize_attributes
     self.uploader_id = CurrentUser.user.id
@@ -22,189 +71,6 @@ class Upload < ApplicationRecord
     self.server = Danbooru.config.server_host
   end
 
-  module ValidationMethods
-    def uploader_is_not_limited
-      if !uploader.can_upload?
-        self.errors.add(:uploader, uploader.upload_limited_reason)
-        return false
-      else
-        return true
-      end
-    end
-
-    def file_or_source_is_present
-      if file.blank? && source.blank?
-        self.errors.add(:base, "Must choose file or specify source")
-        return false
-      else
-        return true
-      end
-    end
-
-    # Because uploads are processed serially, there's no race condition here.
-    def validate_md5_uniqueness
-      md5_post = Post.find_by_md5(md5)
-
-      if md5_post && replaced_post
-        raise "duplicate: #{md5_post.id}" if replaced_post != md5_post
-      elsif md5_post
-        raise "duplicate: #{md5_post.id}"
-      end
-    end
-
-    def validate_file_content_type
-      unless is_valid_content_type?
-        raise "invalid content type (only JPEG, PNG, GIF, SWF, MP4, and WebM files are allowed)"
-      end
-
-      if is_ugoira? && ugoira_service.empty?
-        raise "missing frame data for ugoira"
-      end
-    end
-
-    def validate_md5_confirmation
-      if !md5_confirmation.blank? && md5_confirmation != md5
-        raise "md5 mismatch"
-      end
-    end
-
-    def validate_dimensions
-      resolution = image_width * image_height
-
-      if resolution > Danbooru.config.max_image_resolution
-        raise "image resolution is too large (resolution: #{(resolution / 1_000_000.0).round(1)} megapixels (#{image_width}x#{image_height}); max: #{Danbooru.config.max_image_resolution / 1_000_000} megapixels)"
-      elsif image_width > Danbooru.config.max_image_width
-        raise "image width is too large (width: #{image_width}; max width: #{Danbooru.config.max_image_width})"
-      elsif image_height > Danbooru.config.max_image_height
-        raise "image height is too large (height: #{image_height}; max height: #{Danbooru.config.max_image_height})"
-      end
-    end
-
-    def rating_given
-      if rating.present?
-        return true
-      elsif tag_string =~ /(?:\s|^)rating:([qse])/i
-        self.rating = $1.downcase
-        return true
-      else
-        self.errors.add(:base, "Must specify a rating")
-        return false
-      end
-    end
-
-    def automatic_tags
-      return "" unless Danbooru.config.enable_dimension_autotagging
-
-      tags = []
-      tags << "video_with_sound" if is_video_with_audio?
-      tags << "animated_gif" if is_animated_gif?
-      tags << "animated_png" if is_animated_png?
-      tags.join(" ")
-    end
-
-    def validate_video_duration
-      unless uploader.is_admin?
-        if is_video? && video.duration > 120
-          raise "video must not be longer than 2 minutes"
-        end
-      end
-    end
-  end
-
-  module ConversionMethods
-    def process_upload
-      begin
-        update_attribute(:status, "processing")
-
-        self.source = source.to_s.strip
-        if is_downloadable?
-          self.downloaded_source, self.source, self.file = download_from_source(source, referer_url)
-        elsif self.file.respond_to?(:tempfile)
-          self.file = self.file.tempfile
-        end
-
-        self.file_ext = file_header_to_file_ext(file)
-        self.file_size = file.size
-        self.md5 = Digest::MD5.file(file.path).hexdigest
-
-        validate_file_content_type
-        validate_md5_uniqueness
-        validate_md5_confirmation
-        validate_video_duration
-
-        self.tag_string = "#{tag_string} #{automatic_tags}"
-        self.image_width, self.image_height = calculate_dimensions
-        validate_dimensions
-
-        save
-      end
-    end
-
-    def create_post_from_upload
-      post = convert_to_post
-      distribute_files(post)
-
-      if post.save
-        create_artist_commentary(post) if include_artist_commentary?
-        ugoira_service.save_frame_data(post) if is_ugoira?
-        notify_cropper(post)
-        update_attributes(:status => "completed", :post_id => post.id)
-      else
-        update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
-      end
-
-      post
-    end
-
-    def distribute_files(post)
-      preview_file, sample_file = generate_resizes
-      post.distribute_files(file, sample_file, preview_file)
-    ensure
-      preview_file.try(:close!)
-      sample_file.try(:close!)
-    end
-
-    def process!(force = false)
-      process_upload
-      post = create_post_from_upload
-    rescue Exception => x
-      update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
-      nil
-    ensure
-      file.try(:close!)
-    end
-
-    def ugoira_service
-      @ugoira_service ||= PixivUgoiraService.new
-    end
-
-    def convert_to_post
-      Post.new.tap do |p|
-        p.tag_string = tag_string
-        p.md5 = md5
-        p.file_ext = file_ext
-        p.image_width = image_width
-        p.image_height = image_height
-        p.rating = rating
-        p.source = source
-        p.file_size = file_size
-        p.uploader_id = uploader_id
-        p.uploader_ip_addr = uploader_ip_addr
-        p.parent_id = parent_id
-
-        if !uploader.can_upload_free? || upload_as_pending?
-          p.is_pending = true
-        end
-      end
-    end
-
-    def notify_cropper(post)
-      if ImageCropper.enabled?
-        # ImageCropper.notify(post)
-      end
-    end
-  end
-
   module FileMethods
     def is_image?
       %w(jpg gif png).include?(file_ext)
@@ -218,120 +84,9 @@ class Upload < ApplicationRecord
       %w(webm mp4).include?(file_ext)
     end
 
-    def is_video_with_audio?
-      is_video? && video.audio_channels.present?
-    end
-
     def is_ugoira?
       %w(zip).include?(file_ext)
     end
-
-    def is_animated_gif?
-      return false if file_ext != "gif"
-
-      # Check whether the gif has multiple frames by trying to load the second frame.
-      result = Vips::Image.gifload(file.path, page: 1) rescue $ERROR_INFO
-      if result.is_a?(Vips::Image)
-        true
-      elsif result.is_a?(Vips::Error) && result.message =~ /too few frames in GIF file/
-        false
-      else
-        raise result
-      end
-    end
-
-    def is_animated_png?
-      file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
-    end
-  end
-
-  module ResizerMethods
-    def generate_resizes
-      if is_video?
-        preview_file = generate_video_preview_for(video, Danbooru.config.small_image_width, Danbooru.config.small_image_width)
-      elsif is_ugoira?
-        preview_file = PixivUgoiraConverter.generate_preview(file)
-        sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data)
-      elsif is_image?
-        preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
-
-        if image_width > Danbooru.config.large_image_width
-          sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, image_height, 90)
-        end
-      end
-
-      [preview_file, sample_file]
-    end
-
-    def generate_video_preview_for(video, width, height)
-      dimension_ratio = video.width.to_f / video.height
-      if dimension_ratio > 1
-        height = (width / dimension_ratio).to_i
-      else
-        width = (height * dimension_ratio).to_i
-      end
-
-      output_file = Tempfile.new(binmode: true)
-      video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
-      output_file
-    end
-  end
-
-  module DimensionMethods
-    # Figures out the dimensions of the image.
-    def calculate_dimensions
-      if is_video?
-        [video.width, video.height]
-      elsif is_ugoira?
-        ugoira_service.calculate_dimensions(file.path)
-        [ugoira_service.width, ugoira_service.height]
-      else
-        image_size = ImageSpec.new(file.path)
-        [image_size.width, image_size.height]
-      end
-    end
-  end
-
-  module ContentTypeMethods
-    def is_valid_content_type?
-      file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/
-    end
-
-    def file_header_to_file_ext(file)
-      case File.read(file.path, 16)
-      when /^\xff\xd8/n
-        "jpg"
-      when /^GIF87a/, /^GIF89a/
-        "gif"
-      when /^\x89PNG\r\n\x1a\n/n
-        "png"
-      when /^CWS/, /^FWS/, /^ZWS/
-        "swf"
-      when /^\x1a\x45\xdf\xa3/n
-        "webm"
-      when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
-        "mp4"
-      when /^PK\x03\x04/
-        "zip"
-      else
-        "bin"
-      end
-    end
-  end
-
-  module DownloaderMethods
-    # Determines whether the source is downloadable
-    def is_downloadable?
-      source =~ /^https?:\/\// && file.blank?
-    end
-
-    def download_from_source(source, referer_url = nil)
-      download = Downloads::File.new(source, referer_url: referer_url)
-      file = download.download!
-      ugoira_service.load(download.data)
-
-      [download.downloaded_source, download.source, file]
-    end
   end
 
   module StatusMethods
@@ -347,6 +102,14 @@ class Upload < ApplicationRecord
       status == "completed"
     end
 
+    def is_preprocessed?
+      status == "preprocessed"
+    end
+
+    def is_preprocessing?
+      status == "preprocessing"
+    end
+
     def is_duplicate?
       status =~ /duplicate/
     end
@@ -448,28 +211,27 @@ class Upload < ApplicationRecord
     end
   end
 
-  module ArtistCommentaryMethods
-    def create_artist_commentary(post)
-      post.create_artist_commentary(
-        :original_title => artist_commentary_title,
-        :original_description => artist_commentary_desc
-      )
-    end
-  end
-
-  include ConversionMethods
-  include ValidationMethods
   include FileMethods
-  include ResizerMethods
-  include DimensionMethods
-  include ContentTypeMethods
-  include DownloaderMethods
   include StatusMethods
   include UploaderMethods
   include VideoMethods
   extend SearchMethods
   include ApiMethods
-  include ArtistCommentaryMethods
+
+  def uploader_is_not_limited
+    if !uploader.can_upload?
+      self.errors.add(:uploader, uploader.upload_limited_reason)
+      return false
+    else
+      return true
+    end
+  end
+
+  def assign_rating_from_tags
+    if tag_string =~ /(?:\s|^)rating:([qse])/i
+      self.rating = $1.downcase
+    end
+  end
 
   def presenter
     @presenter ||= UploadPresenter.new(self)
@@ -478,8 +240,4 @@ class Upload < ApplicationRecord
   def upload_as_pending?
     as_pending.to_s.truthy?
   end
-
-  def include_artist_commentary?
-    include_artist_commentary.to_s.truthy?
-  end
 end

app/views/uploads/show.html.erb

@@ -12,7 +12,7 @@
       <p>This upload has finished processing. <%= link_to "View the post", post_path(@upload.post_id) %>.</p>
     <% elsif @upload.is_pending? %>
       <p>This upload is waiting to be processed. Please wait a few seconds.</p>
-    <% elsif @upload.is_processing? %>
+    <% elsif @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
       <p>This upload is being processed. Please wait a few seconds.</p>
     <% elsif @upload.is_duplicate? %>
       <p>This upload is a duplicate: <%= link_to "post ##{@upload.duplicate_post_id}", post_path(@upload.duplicate_post_id) %></p>
@@ -42,7 +42,7 @@
   Upload - <%= Danbooru.config.app_name %>
 <% end %>
 
-<% if @upload.is_pending? || @upload.is_processing? %>
+<% if @upload.is_pending? || @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
   <% content_for(:html_header) do %>
     <meta http-equiv="refresh" content="2">
   <% end %>

config/initializers/ffmpeg.rb

@@ -0,0 +1,3 @@
+unless Rails.env.development?
+  FFMPEG.logger.level = Logger::ERROR
+end

db/migrate/20180517190048_add_missing_fields_to_uploads.rb

@@ -0,0 +1,12 @@
+class AddMissingFieldsToUploads < ActiveRecord::Migration[5.2]
+  def change
+    add_column :uploads, :md5, :string
+    add_column :uploads, :file_ext, :string
+    add_column :uploads, :file_size, :integer
+    add_column :uploads, :image_width, :integer
+    add_column :uploads, :image_height, :integer
+    add_column :uploads, :artist_commentary_desc, :text
+    add_column :uploads, :artist_commentary_title, :text
+    add_column :uploads, :include_artist_commentary, :boolean
+  end
+end

db/migrate/20180518175154_add_context_to_uploads.rb

@@ -0,0 +1,5 @@
+class AddContextToUploads < ActiveRecord::Migration[5.2]
+  def change
+    add_column :uploads, :context, :text
+  end
+end

test/factories/upload.rb

@@ -14,6 +14,14 @@ FactoryBot.define do
       source "http://www.google.com/intl/en_ALL/images/logo.gif"
     end
 
+    factory(:ugoira_upload) do
+      file do
+        f = Tempfile.new
+        IO.copy_stream("#{Rails.root}/test/fixtures/ugoira.zip", f.path)
+        ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "ugoira.zip")
+      end
+    end
+
     factory(:jpg_upload) do
       file do
         f = Tempfile.new

test/functional/uploads_controller_test.rb

@@ -38,6 +38,15 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
         assert_response :success
       end
 
+      context "with a url" do
+        should "preprocess" do
+          assert_difference(-> { Upload.count }) do
+            get_auth new_upload_path, @user, params: {:url => "http://www.google.com/intl/en_ALL/images/logo.gif"}
+            assert_response :success
+          end
+        end
+      end
+
       context "for a twitter post" do
         should "render" do
           skip "Twitter keys are not set" unless Danbooru.config.twitter_api_key
@@ -49,13 +58,15 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
       context "for a post that has already been uploaded" do
         setup do
           as_user do
-            @post = create(:post, :source => "aaa")
+            @post = create(:post, :source => "http://google.com/aaa")
           end
         end
 
         should "initialize the post" do
-          get_auth new_upload_path, @user, params: {:url => "http://google.com/aaa"}
+          assert_difference(-> { Upload.count }, 0) do
-          assert_response :success
+            get_auth new_upload_path, @user, params: {:url => "http://google.com/aaa"}
+            assert_response :success
+          end
         end
       end
     end

test/models/upload_service_test.rb

@@ -0,0 +1,525 @@
+require 'test_helper'
+
+class UploadServiceTest < ActiveSupport::TestCase
+  UGOIRA_CONTEXT = {
+    "ugoira" => {
+      "frame_data" => [
+        {"file" => "000000.jpg", "delay" => 200},
+        {"file" => "000001.jpg", "delay" => 200},
+        {"file" => "000002.jpg", "delay" => 200},
+        {"file" => "000003.jpg", "delay" => 200},
+        {"file" => "000004.jpg", "delay" => 250}
+      ],
+      "content_type" => "image/jpeg"
+    }
+  }.freeze
+
+  context "::Utils" do
+    subject { UploadService::Utils }
+
+    context ".calculate_ugoira_dimensions" do
+      context "for a valid ugoira file" do
+        setup do
+          @path = "test/files/valid_ugoira.zip"
+        end
+
+        should "extract the dimensions" do
+          w, h = subject.calculate_ugoira_dimensions(@path)
+          assert_operator(w, :>, 0)
+          assert_operator(h, :>, 0)
+        end
+      end
+
+      context "for an invalid ugoira file" do
+        setup do
+          @path = "test/files/invalid_ugoira.zip"
+        end
+
+        should "raise an error" do
+          assert_raises(ImageSpec::Error) do
+            subject.calculate_ugoira_dimensions(@path)
+          end
+        end
+      end
+    end
+
+    context ".calculate_dimensions" do
+      context "for an ugoira" do
+        setup do
+          @file = File.open("test/files/valid_ugoira.zip", "rb")
+          @upload = mock()
+          @upload.stubs(:is_video?).returns(false)
+          @upload.stubs(:is_ugoira?).returns(true)
+        end
+
+        teardown do
+          @file.close
+        end
+
+        should "return the dimensions" do
+          subject.expects(:calculate_ugoira_dimensions).once.returns([60, 60])
+          subject.calculate_dimensions(@upload, @file) do |w, h|
+            assert_operator(w, :>, 0)
+            assert_operator(h, :>, 0)
+          end
+        end
+      end
+
+      context "for a video" do
+        setup do
+          @file = File.open("test/files/test-300x300.mp4", "rb")
+          @upload = mock()
+          @upload.stubs(:is_video?).returns(true)
+        end
+
+        teardown do
+          @file.close
+        end
+
+        should "return the dimensions" do
+          subject.calculate_dimensions(@upload, @file) do |w, h|
+            assert_operator(w, :>, 0)
+            assert_operator(h, :>, 0)
+          end
+        end
+      end
+
+      context "for an image" do 
+        setup do
+          @file = File.open("test/files/test.jpg", "rb")
+          @upload = mock()
+          @upload.stubs(:is_video?).returns(false)
+          @upload.stubs(:is_ugoira?).returns(false)
+        end
+
+        teardown do
+          @file.close
+        end
+
+        should "find the dimensions" do
+          subject.calculate_dimensions(@upload, @file) do |w, h|
+            assert_operator(w, :>, 0)
+            assert_operator(h, :>, 0)
+          end
+        end
+      end
+    end
+
+    context ".process_file" do
+      setup do
+        @upload = FactoryBot.build(:jpg_upload)
+        @file = @upload.file
+      end
+
+      should "run" do
+        subject.expects(:distribute_files).twice
+        subject.process_file(@upload, @file)
+        assert_equal("jpg", @upload.file_ext)
+        assert_equal(28086, @upload.file_size)
+        assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
+        assert_equal(335, @upload.image_height)
+        assert_equal(500, @upload.image_width)
+      end
+    end
+
+    context ".generate_resizes" do
+      context "for an ugoira" do
+        setup do
+          context = UGOIRA_CONTEXT
+          @file = File.open("test/fixtures/ugoira.zip", "rb")
+          @upload = mock()
+          @upload.stubs(:is_video?).returns(false)
+          @upload.stubs(:is_ugoira?).returns(true)
+          @upload.stubs(:context).returns(context)
+        end
+
+        should "generate a preview and a video" do
+          preview, sample = subject.generate_resizes(@file, @upload)
+          assert_operator(File.size(preview.path), :>, 0)
+          assert_operator(File.size(sample.path), :>, 0)
+          preview.close
+          preview.unlink
+          sample.close
+          sample.unlink
+        end
+      end
+
+      context "for a video" do
+        teardown do
+          @file.close
+        end
+
+        context "for an mp4" do
+          setup do
+            @file = File.open("test/files/test-300x300.mp4", "rb")
+            @upload = mock()
+            @upload.stubs(:is_video?).returns(true)
+            @upload.stubs(:is_ugoira?).returns(false)
+          end
+
+          should "generate a video" do
+            preview, sample = subject.generate_resizes(@file, @upload)
+            assert_operator(File.size(preview.path), :>, 0)
+            preview.close
+            preview.unlink
+          end
+        end
+
+        context "for a webm" do
+          setup do
+            @file = File.open("test/files/test-512x512.webm", "rb")
+            @upload = mock()
+            @upload.stubs(:is_video?).returns(true)
+            @upload.stubs(:is_ugoira?).returns(false)
+          end
+
+          should "generate a video" do
+            preview, sample = subject.generate_resizes(@file, @upload)
+            assert_operator(File.size(preview.path), :>, 0)
+            preview.close
+            preview.unlink
+          end
+        end
+      end
+
+      context "for an image" do
+        teardown do
+          @file.close
+        end
+
+        setup do
+          @upload = mock()
+          @upload.stubs(:is_video?).returns(false)
+          @upload.stubs(:is_ugoira?).returns(false)
+          @upload.stubs(:is_image?).returns(true)
+          @upload.stubs(:image_width).returns(1200)
+          @upload.stubs(:image_height).returns(200)
+        end
+
+        context "for a jpeg" do
+          setup do
+            @file = File.open("test/files/test.jpg", "rb")
+          end
+
+          should "generate a preview" do
+            preview, sample = subject.generate_resizes(@file, @upload)
+            assert_operator(File.size(preview.path), :>, 0)
+            assert_operator(File.size(sample.path), :>, 0)
+            preview.close
+            preview.unlink
+            sample.close
+            sample.unlink
+          end
+        end
+
+        context "for a png" do
+          setup do
+            @file = File.open("test/files/test.png", "rb")
+          end
+
+          should "generate a preview" do
+            preview, sample = subject.generate_resizes(@file, @upload)
+            assert_operator(File.size(preview.path), :>, 0)
+            assert_operator(File.size(sample.path), :>, 0)
+            preview.close
+            preview.unlink
+            sample.close
+            sample.unlink
+          end
+        end
+
+        context "for a gif" do
+          setup do
+            @file = File.open("test/files/test.png", "rb")
+          end
+
+          should "generate a preview" do
+            preview, sample = subject.generate_resizes(@file, @upload)
+            assert_operator(File.size(preview.path), :>, 0)
+            assert_operator(File.size(sample.path), :>, 0)
+            preview.close
+            preview.unlink
+            sample.close
+            sample.unlink
+          end
+        end
+      end
+    end
+
+    context ".generate_video_preview_for" do
+      context "for an mp4" do
+        setup do
+          @path = "test/files/test-300x300.mp4"
+          @video = FFMPEG::Movie.new(@path)
+        end
+
+        should "generate a video" do
+          sample = subject.generate_video_preview_for(@video, 100, 100)
+          assert_operator(File.size(sample.path), :>, 0)
+          sample.close
+          sample.unlink
+        end
+      end
+
+      context "for a webm" do
+        setup do
+          @path = "test/files/test-512x512.webm"
+          @video = FFMPEG::Movie.new(@path)
+        end
+
+        should "generate a video" do
+          sample = subject.generate_video_preview_for(@video, 100, 100)
+          assert_operator(File.size(sample.path), :>, 0)
+          sample.close
+          sample.unlink
+        end
+      end
+    end
+  end
+
+  context "::Preprocessor" do
+    subject { UploadService::Preprocessor }
+
+    context "#download_from_source" do
+      setup do
+        @jpeg = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg"
+        @ugoira = "https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip"
+      end
+
+      should "work on a jpeg" do
+        file = subject.new({}).download_from_source(@jpeg) do |context|
+          assert_not_nil(context[:downloaded_source])
+          assert_not_nil(context[:source])
+        end
+
+        assert_operator(File.size(file.path), :>, 0)
+        file.close
+      end
+
+      should "work on an ugoira url" do
+        file = subject.new({}).download_from_source(@ugoira, referer_url: "https://www.pixiv.net") do |context|
+          assert_not_nil(context[:downloaded_source])
+          assert_not_nil(context[:source])
+          assert_not_nil(context[:ugoira])
+        end
+
+        assert_operator(File.size(file.path), :>, 0)
+        file.close
+      end
+    end
+
+    context "#start!" do
+      setup do
+        CurrentUser.user = travel_to(1.month.ago) do
+          FactoryBot.create(:user)
+        end
+        CurrentUser.ip_addr = "127.0.0.1"
+        @jpeg = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg"
+        @ugoira = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364"
+        @video = "https://www.sample-videos.com/video/mp4/720/big_buck_bunny_720p_1mb.mp4"
+      end
+
+      teardown do
+        CurrentUser.user = nil
+        CurrentUser.ip_addr = nil
+      end
+
+      should "work for a jpeg" do
+        @service = subject.new(source: @jpeg)
+        @upload = @service.start!
+        assert_equal("preprocessed", @upload.status)
+        assert_not_nil(@upload.md5)
+        assert_equal("jpg", @upload.file_ext)
+        assert_operator(@upload.file_size, :>, 0)
+        assert_not_nil(@upload.source)
+        assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :original)))
+        assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :large)))
+        assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :preview)))
+      end
+
+      should "work for an ugoira" do
+        @service = subject.new(source: @ugoira)
+        @upload = @service.start!
+        assert_equal("preprocessed", @upload.status)
+        assert_not_nil(@upload.md5)
+        assert_equal("zip", @upload.file_ext)
+        assert_operator(@upload.file_size, :>, 0)
+        assert_not_nil(@upload.source)
+        assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "zip", :original)))
+        assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "zip", :large)))
+      end
+
+      should "work for a video" do
+        @service = subject.new(source: @video)
+        @upload = @service.start!
+        assert_equal("preprocessed", @upload.status)
+        assert_not_nil(@upload.md5)
+        assert_equal("mp4", @upload.file_ext)
+        assert_operator(@upload.file_size, :>, 0)
+        assert_not_nil(@upload.source)
+        assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "mp4", :original)))
+        assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "mp4", :preview)))
+      end
+
+      context "on timeout errors" do
+        setup do
+          HTTParty.stubs(:get).raises(Net::ReadTimeout)
+        end
+        
+        should "leave the upload in an error state" do
+          @service = subject.new(source: @video)
+          @upload = @service.start!
+          assert_match(/error:/, @upload.status)
+        end
+      end
+
+    end
+  end
+
+  context "#start!" do
+    subject { UploadService }
+
+    setup do
+      @source = "https://upload.wikimedia.org/wikipedia/commons/c/c5/Moraine_Lake_17092005.jpg"
+      CurrentUser.user = travel_to(1.month.ago) do
+        FactoryBot.create(:user)
+      end
+      CurrentUser.ip_addr = "127.0.0.1"
+    end
+
+    teardown do
+      CurrentUser.user = nil
+      CurrentUser.ip_addr = nil
+    end
+
+    context "automatic tagging" do
+      setup do
+        @build_service = ->(file) { subject.new(file: file)}
+      end
+
+      should "tag animated png files" do
+        service = @build_service.call(upload_file("test/files/apng/normal_apng.png"))
+        upload = service.start!
+        assert_match(/animated_png/, upload.tag_string)
+      end
+
+      should "tag animated gif files" do
+        service = @build_service.call(upload_file("test/files/test-animated-86x52.gif"))
+        upload = service.start!
+        assert_match(/animated_gif/, upload.tag_string)
+      end
+
+      should "not tag static gif files" do
+        service = @build_service.call(upload_file("test/files/test-static-32x32.gif"))
+        upload = service.start!
+        assert_no_match(/animated_gif/, upload.tag_string)
+      end
+    end
+
+    context "that is too large" do
+      setup do
+        Danbooru.config.stubs(:max_image_resolution).returns(31*31)
+      end
+
+      should "should fail validation" do
+        service = subject.new(file: upload_file("test/files/test-static-32x32.gif"))
+        upload = service.start!
+        assert_match(/image resolution is too large/, upload.status)
+      end
+    end
+
+    context "with a preprocessing predecessor" do
+      setup do
+        @predecessor = FactoryBot.create(:source_upload, status: "preprocessing", source: @source, image_height: 0, image_width: 0, file_ext: "jpg")
+        Delayed::Worker.delay_jobs = true
+      end
+
+      teardown do
+        Delayed::Worker.delay_jobs = false
+      end
+
+      should "schedule a job later" do
+        service = subject.new(source: @source)
+
+        assert_difference(-> { Delayed::Job.count }) do
+          predecessor = service.start!
+          assert_equal(@predecessor, predecessor)
+        end
+      end
+    end
+
+    context "with a preprocessed predecessor" do
+      setup do
+        @predecessor = FactoryBot.create(:source_upload, status: "preprocessed", source: @source, image_height: 0, image_width: 0, file_ext: "jpg")
+        @tags = 'hello world'
+      end
+
+      should "update the predecessor" do
+        service = subject.new(source: @source, tag_string: @tags)
+
+        predecessor = service.start!
+        assert_equal(@predecessor, predecessor)
+        assert_equal(@tags, predecessor.tag_string.strip)
+      end
+    end
+
+    context "with no predecessor" do
+      should "create an upload" do
+        service = subject.new(source: @source)
+
+        assert_difference(-> { Upload.count }) do
+          service.start!
+        end
+      end
+    end
+  end
+
+  context "#create_post_from_upload" do
+    subject { UploadService }
+
+    setup do
+      CurrentUser.user = travel_to(1.month.ago) do
+        FactoryBot.create(:user)
+      end
+      CurrentUser.ip_addr = "127.0.0.1"
+    end
+
+    teardown do
+      CurrentUser.user = nil
+      CurrentUser.ip_addr = nil
+    end
+
+    context "for an ugoira" do
+      setup do
+        @upload = FactoryBot.create(:ugoira_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, context: UGOIRA_CONTEXT)
+      end
+
+      should "create a post" do
+        assert_difference(-> { PixivUgoiraFrameData.count }) do
+          post = subject.new({}).create_post_from_upload(@upload)
+          assert_equal([], post.errors.full_messages)
+          assert_not_nil(post.id)
+        end
+      end
+    end
+
+    context "for an image" do
+      setup do
+        @upload = FactoryBot.create(:source_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100)
+      end
+
+      should "create a commentary record" do
+        assert_difference(-> { ArtistCommentary.count }) do
+          subject.new({include_artist_commentary: true, artist_commentary_title: "blah", artist_commentary_desc: "blah"}).create_post_from_upload(@upload)
+        end
+      end
+
+      should "create a post" do
+        post = subject.new({}).create_post_from_upload(@upload)
+        assert_equal([], post.errors.full_messages)
+        assert_not_nil(post.id)
+      end
+    end
+
+  end
+end

test/unit/upload_test.rb

@@ -1,6 +1,8 @@
 require 'test_helper'
 
 class UploadTest < ActiveSupport::TestCase
+  SOURCE_URL = "https://upload.wikimedia.org/wikipedia/commons/thumb/6/66/NAMA_Machine_d%27Anticyth%C3%A8re_1.jpg/538px-NAMA_Machine_d%27Anticyth%C3%A8re_1.jpg?download"
+
   context "In all cases" do
     setup do
       mock_iqdb_service!
@@ -28,298 +30,12 @@ class UploadTest < ActiveSupport::TestCase
         end
       end
 
-      context "image size calculator" do
-        should "discover the dimensions for a compressed SWF" do
-          @upload = FactoryBot.create(:upload, file: upload_file("test/files/compressed.swf"))
-          assert_equal([607, 756], @upload.calculate_dimensions)
-        end
-
-        should "discover the dimensions for a JPG with JFIF data" do
-          @upload = FactoryBot.create(:jpg_upload)
-          assert_equal([500, 335], @upload.calculate_dimensions)
-        end
-
-        should "discover the dimensions for a JPG with EXIF data" do
-          @upload = FactoryBot.create(:upload, file: upload_file("test/files/test-exif-small.jpg"))
-          assert_equal([529, 600], @upload.calculate_dimensions)
-        end
-
-        should "discover the dimensions for a JPG with no header data" do
-          @upload = FactoryBot.create(:upload, file: upload_file("test/files/test-blank.jpg"))
-          assert_equal([668, 996], @upload.calculate_dimensions)
-        end
-
-        should "discover the dimensions for a PNG" do
-          @upload = FactoryBot.create(:upload, file: upload_file("test/files/test.png"))
-          assert_equal([768, 1024], @upload.calculate_dimensions)
-        end
-
-        should "discover the dimensions for a GIF" do
-          @upload = FactoryBot.create(:upload, file: upload_file("test/files/test.gif"))
-          assert_equal([400, 400], @upload.calculate_dimensions)
-        end
-      end
-
-      context "content type calculator" do
-        should "know how to parse jpeg, png, gif, and swf file headers" do
-          @upload = FactoryBot.create(:jpg_upload)
-          assert_equal("jpg", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.jpg")))
-          assert_equal("gif", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.gif")))
-          assert_equal("png", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.png")))
-          assert_equal("swf", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/compressed.swf")))
-          assert_equal("bin", @upload.file_header_to_file_ext(File.open("#{Rails.root}/README.md")))
-        end
-      end
-
-      context "downloader" do
-        context "for a zip that is not an ugoira" do
-          should "not validate" do
-            @upload = FactoryBot.create(:upload, file: upload_file("test/files/invalid_ugoira.zip"))
-            @upload.process!
-            assert_equal("error: RuntimeError - missing frame data for ugoira", @upload.status)
-          end
-        end
-
-        context "that is a pixiv ugoira" do
-          setup do
-            @url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654"
-            @upload = FactoryBot.create(:source_upload, :source => @url, :tag_string => "ugoira")
-          end
-
-          should "process successfully" do
-            begin
-              _, _, output_file = @upload.download_from_source(@url, "")
-              assert_operator(output_file.size, :>, 1_000)
-              assert_equal("zip", @upload.file_header_to_file_ext(output_file))
-            rescue Net::OpenTimeout
-              skip "Remote connection to #{@url} failed"
-            end
-          end
-        end
-      end
-
-      context "determining if a file is downloadable" do
-        should "classify HTTP sources as downloadable" do
-          @upload = FactoryBot.create(:source_upload, :source => "http://www.google.com/1.jpg")
-          assert_not_nil(@upload.is_downloadable?)
-        end
-
-        should "classify HTTPS sources as downloadable" do
-          @upload = FactoryBot.create(:source_upload, :source => "https://www.google.com/1.jpg")
-          assert_not_nil(@upload.is_downloadable?)
-        end
-
-        should "classify non-HTTP/HTTPS sources as not downloadable" do
-          @upload = FactoryBot.create(:source_upload, :source => "ftp://www.google.com/1.jpg")
-          assert_nil(@upload.is_downloadable?)
-        end
-      end
-
-      context "file processor" do
-        should "parse and process a cgi file representation" do
-          @upload = FactoryBot.create(:upload, file: upload_file("test/files/test.jpg"))
-          assert_nothing_raised {@upload.process_upload}
-          assert_equal(28086, @upload.file_size)
-        end
-
-        should "process a transparent png" do
-          @upload = FactoryBot.create(:upload, file: upload_file("test/files/alpha.png"))
-          assert_nothing_raised {@upload.process_upload}
-          assert_equal(1136, @upload.file_size)
-        end
-      end
-
-      context "hash calculator" do
-        should "caculate the hash" do
-          @upload = FactoryBot.create(:jpg_upload)
-          @upload.process_upload
-          assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
-        end
-      end
-
-      context "resizer" do
-        should "generate several resized versions of the image" do
-          @upload = FactoryBot.create(:upload, file_ext: "jpg", image_width: 1356, image_height: 911, file: upload_file("test/files/test-large.jpg"))
-          preview_file, sample_file = @upload.generate_resizes
-          assert_operator(preview_file.size, :>, 1_000)
-          assert_operator(sample_file.size, :>, 1_000)
-        end
-      end
-
       should "increment the uploaders post_upload_count" do
-        @upload = FactoryBot.create(:source_upload)
+        assert_difference(-> { CurrentUser.user.post_upload_count }) do
-        assert_difference("CurrentUser.user.post_upload_count", 1) do
+          FactoryBot.create(:source_upload)
-          @upload.process!
           CurrentUser.user.reload
         end
       end
-
-      context "with an artist commentary" do
-        setup do
-          @upload = FactoryBot.create(:source_upload,
-            include_artist_commentary: "1",
-            artist_commentary_title: "",
-            artist_commentary_desc: "blah",
-          )
-        end
-
-        should "create an artist commentary when processed" do
-          assert_difference("ArtistCommentary.count") do
-            @upload.process!
-          end
-        end
-      end
-
-      should "process completely for a downloaded image" do
-        @upload = FactoryBot.create(:source_upload,
-          :rating => "s",
-          :uploader_ip_addr => "127.0.0.1",
-          :tag_string => "hoge foo"
-          )
-        assert_difference("Post.count") do
-          assert_nothing_raised {@upload.process!}
-        end
-
-        post = Post.last
-        assert_equal("http://www.google.com/intl/en_ALL/images/logo.gif", post.source)
-        assert_equal("foo hoge lowres", post.tag_string)
-        assert_equal("s", post.rating)
-        assert_equal(@upload.uploader_id, post.uploader_id)
-        assert_equal("127.0.0.1", post.uploader_ip_addr.to_s)
-        assert_equal(@upload.md5, post.md5)
-        assert_equal("gif", post.file_ext)
-        assert_equal(276, post.image_width)
-        assert_equal(110, post.image_height)
-        assert_equal(8558, post.file_size)
-        assert_equal(post.id, @upload.post_id)
-        assert_equal("completed", @upload.status)
-      end
-
-      context "automatic tagging" do
-        should "tag animated png files" do
-          @upload = FactoryBot.build(:upload, file_ext: "png", file: upload_file("test/files/apng/normal_apng.png"))
-          assert_equal("animated_png", @upload.automatic_tags)
-        end
-
-        should "tag animated gif files" do
-          @upload = FactoryBot.build(:upload, file_ext: "gif", file: upload_file("test/files/test-animated-86x52.gif"))
-          assert_equal("animated_gif", @upload.automatic_tags)
-        end
-
-        should "not tag static gif files" do
-          @upload = FactoryBot.build(:upload, file_ext: "gif", file: upload_file("test/files/test-static-32x32.gif"))
-          assert_equal("", @upload.automatic_tags)
-        end
-      end
-
-      context "that is too large" do
-        should "should fail validation" do
-          Danbooru.config.stubs(:max_image_resolution).returns(31*31)
-          @upload = FactoryBot.create(:upload, file: upload_file("test/files/test-static-32x32.gif"))
-          @upload.process!
-          assert_match(/image resolution is too large/, @upload.status)
-        end
-      end
-    end
-
-    should "process completely for a pixiv ugoira" do
-      skip "ffmpeg is not installed" unless check_ffmpeg
-      @upload = FactoryBot.create(:source_upload, source: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654")
-
-      assert_difference(["PixivUgoiraFrameData.count", "Post.count"]) do
-        @upload.process!
-        assert_equal([], @upload.errors.full_messages)
-      end
-      post = Post.last
-      assert_not_nil(post.pixiv_ugoira_frame_data)
-      assert_equal("0d94800c4b520bf3d8adda08f95d31e2", post.md5)
-      assert_equal(60, post.image_width)
-      assert_equal(60, post.image_height)
-      assert_equal("https://i.pximg.net/img-zip-ugoira/img/2014/10/05/23/42/23/46378654_ugoira1920x1080.zip", post.source)
-      assert_nothing_raised { post.file(:original) }
-      assert_nothing_raised { post.file(:preview) }
-    end
-
-    should "process completely for an uploaded image" do
-      @upload = FactoryBot.create(:jpg_upload,
-        :rating => "s",
-        :uploader_ip_addr => "127.0.0.1",
-        :tag_string => "hoge foo",
-        :file => upload_file("test/files/test.jpg"),
-        )
-
-      assert_difference("Post.count") do
-        assert_nothing_raised {@upload.process!}
-      end
-      post = Post.last
-      assert_equal("foo hoge lowres", post.tag_string)
-      assert_equal("s", post.rating)
-      assert_equal(@upload.uploader_id, post.uploader_id)
-      assert_equal("127.0.0.1", post.uploader_ip_addr.to_s)
-      assert_equal(@upload.md5, post.md5)
-      assert_equal("jpg", post.file_ext)
-      assert_nothing_raised { post.file(:original) }
-      assert_equal(28086, post.file(:original).size)
-      assert_equal(post.id, @upload.post_id)
-      assert_equal("completed", @upload.status)
-    end
-
-    should "process completely for a .webm" do
-      upload = FactoryBot.create(:upload, rating: "s", file: upload_file("test/files/test-512x512.webm"))
-
-      assert_difference("Post.count") do
-        upload.process!
-        assert_equal("completed", upload.status)
-      end
-
-      post = Post.last
-      assert_includes(post.tag_array, "webm")
-      assert_equal("webm", upload.file_ext)
-      assert_equal(12345, upload.file_size)
-      assert_equal(512, upload.image_width)
-      assert_equal(512, upload.image_height)
-      assert_equal("34dd2489f7aaa9e57eda1b996ff26ff7", upload.md5)
-
-      assert_nothing_raised { post.file(:preview) }
-      assert_nothing_raised { post.file(:original) }
-    end
-
-    should "process completely for a .mp4" do
-      upload = FactoryBot.create(:upload, rating: "s", file: upload_file("test/files/test-300x300.mp4"))
-
-      assert_difference("Post.count") do
-        upload.process!
-        assert_equal("completed", upload.status)
-      end
-
-      post = Post.last
-      assert_includes(post.tag_array, "mp4")
-      assert_equal("mp4", upload.file_ext)
-      assert_equal(18677, upload.file_size)
-      assert_equal(300, upload.image_width)
-      assert_equal(300, upload.image_height)
-      assert_equal("865c93102cad3e8a893d6aac6b51b0d2", upload.md5)
-
-      assert_nothing_raised { post.file(:preview) }
-      assert_nothing_raised { post.file(:original) }
-    end
-
-    should "process completely for a null source" do
-      @upload = FactoryBot.create(:jpg_upload, :source => nil)
-
-      assert_difference("Post.count") do
-        assert_nothing_raised {@upload.process!}
-      end
-    end
-
-    context "on timeout errors" do
-      should "leave the upload in an error state" do
-        HTTParty.stubs(:get).raises(Net::ReadTimeout)
-        @upload = FactoryBot.create(:source_upload)
-        @upload.process!
-
-        assert_match(/\Aerror/, @upload.status)
-      end
     end
   end
 end