diff --git a/app/models/upload.rb b/app/models/upload.rb new file mode 100644 index 000000000..cdf5e3e7d --- /dev/null +++ b/app/models/upload.rb @@ -0,0 +1,210 @@ +require "danbooru_image_resizer/danbooru_image_resizer" +require "tmpdir" + +class Upload < ActiveRecord::Base + attr_accessor :file, :image_width, :image_height, :file_ext, :md5, :file_size + belongs_to :uploader, :class_name => "User" + before_save :convert_cgi_file + + def process! + update_attribute(:status, "processing") + if is_downloadable? + download_from_source(temp_file_path) + end + self.file_ext = content_type_to_file_ext(content_type) + calculate_hash(file_path) + calculate_file_size(file_path) + calculate_dimensions(file_path) if has_dimensions? + generate_resizes(file_path) + move_file + post = convert_to_post + if post.save + update_attributes(:status => "finished", :post_id => post.id) + else + update_attribute(:status, "error: " + post.errors.full_messages.join(", ")) + end + end + + def convert_to_post + returning Post.new do |p| + p.tag_string = tag_string + p.md5 = md5 + p.file_ext = file_ext + p.image_width = image_width + p.image_height = image_height + p.uploader_id = uploader_id + p.uploader_ip_addr = uploader_ip_addr + p.updater_id = uploader_id + p.updater_ip_addr = uploader_ip_addr + p.rating = rating + p.source = source + p.file_size = file_size + end + end + + def move_file + FileUtils.mv(file_path, md5_file_path) + end + + def calculate_file_size(source_path) + self.file_size = File.size(source_path) + end + + # Calculates the MD5 based on whatever is in temp_file_path + def calculate_hash(source_path) + self.md5 = Digest::MD5.file(source_path).hexdigest + end + + class Error < Exception ; end + + module ResizerMethods + def generate_resizes(source_path) + generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path) + generate_resize_for(Danbooru.config.medium_image_width, nil, source_path) + generate_resize_for(Danbooru.config.large_image_width, nil, source_path) + end + + def generate_resize_for(width, height, source_path) + return if width.nil? + return unless image_width > width + return unless height.nil? || image_height > height + + unless File.exists?(source_path) + raise Error.new("file not found") + end + + size = Danbooru.reduce_to({:width => image_width, :height => image_height}, {:width => width, :height => height}) + + # If we're not reducing the resolution, only reencode if the source image larger than + # 200 kilobytes. + if size[:width] == image_width && size[:height] == image_height && File.size?(source_path) < 200.kilobytes + return + end + + Danbooru.resize(file_ext, source_path, resized_file_path_for(width), size, 90) + end + end + + module DimensionMethods + # Figures out the dimensions of the image. + def calculate_dimensions(file_path) + image_size = ImageSize.new(File.open(file_path, "rb")) + self.image_width = image_size.get_width + self.image_height = image_size.get_height + end + + # Does this file have image dimensions? + def has_dimensions? + %w(jpg gif png swf).include?(file_ext) + end + end + + module ContentTypeMethods + def content_type_to_file_ext(content_type) + case content_type + when "image/jpeg" + "jpg" + + when "image/gif" + "gif" + + when "image/png" + "png" + + when "application/x-shockwave-flash" + "swf" + + else + "bin" + end + end + + # Converts a content type string to a file extension + def file_ext_to_content_type(file_ext) + case file_ext + when /\.jpeg$|\.jpg$/ + "image/jpeg" + + when /\.gif$/ + "image/gif" + + when /\.png$/ + "image/png" + + when /\.swf$/ + "application/x-shockwave-flash" + + else + "application/octet-stream" + end + end + end + + module FilePathMethods + def md5_file_path + prefix = Rails.env == "test" ? "test." : "" + "#{Rails.root}/public/data/original/#{prefix}#{md5}.#{file_ext}" + end + + def resized_file_path_for(width) + prefix = Rails.env == "test" ? "test." : "" + + case width + when Danbooru.config.small_image_width + "#{Rails.root}/public/data/thumb/#{prefix}#{md5}.jpg" + + when Danbooru.config.medium_image_width + "#{Rails.root}/public/data/medium/#{prefix}#{md5}.jpg" + + when Danbooru.config.large_image_width + "#{Rails.root}/public/data/large/#{prefix}#{md5}.jpg" + end + end + + def temp_file_path + File.join(Dir::tmpdir, "#{Time.now.to_f}.#{$PROCESS_ID}") + end + end + + module DownloaderMethods + # Determines whether the source is downloadable + def is_downloadable? + source =~ /^http:\/\// && file_path.blank? + end + + # Downloads the file to destination_path + def download_from_source(destination_path) + download = Download.new(source, destination_path) + download.download! + self.file_path = destination_path + self.content_type = download.content_type || file_ext_to_content_type(source) + self.file_ext = content_type_to_file_ext(content_type) + self.source = download.source + end + end + + module CgiFileMethods + def convert_cgi_file + return if file.blank? || file.size == 0 + + if file.local_path + self.file_path = file.local_path + else + self.file_path = temp_file_path + + File.open(file_path, 'wb') do |out| + out.write(file.read) + end + end + self.content_type = file.content_type || file_ext_to_content_type(file.original_filename) + self.file_ext = content_type_to_file_ext(content_type) + end + end + + include ResizerMethods + include DimensionMethods + include ContentTypeMethods + include DownloaderMethods + include FilePathMethods + include CgiFileMethods +end \ No newline at end of file diff --git a/db/migrate/20100205224030_create_uploads.rb b/db/migrate/20100205224030_create_uploads.rb new file mode 100644 index 000000000..f54fa76f5 --- /dev/null +++ b/db/migrate/20100205224030_create_uploads.rb @@ -0,0 +1,21 @@ +class CreateUploads < ActiveRecord::Migration + def self.up + create_table :uploads do |t| + t.timestamps + t.column :source, :string + t.column :file_path, :string + t.column :content_type, :string + + t.column :rating, :character, :null => false + t.column :uploader_id, :integer, :null => false + t.column :uploader_ip_addr, "inet", :null => false + t.column :tag_string, :text, :null => false + t.column :status, :string, :null => false, :default => "pending" + t.column :post_id, :integer + end + end + + def self.down + drop_table :uploads + end +end diff --git a/test/factories/upload.rb b/test/factories/upload.rb new file mode 100644 index 000000000..d4eaa56f6 --- /dev/null +++ b/test/factories/upload.rb @@ -0,0 +1,43 @@ +require 'fileutils' + +Factory.define(:upload) do |f| + f.rating "s" + f.uploader {|x| x.association(:user)} + f.uploader_ip_addr "127.0.0.1" + f.tag_string "special" + f.status "pending" +end + +Factory.define(:downloadable_upload, :parent => :upload) do |f| + f.source "http://www.google.com/intl/en_ALL/images/logo.gif" +end + +Factory.define(:uploaded_jpg_upload, :parent => :upload) do |f| + f.file_path do + FileUtils.cp("#{Rails.root}/test/files/test.jpg", "#{Rails.root}/tmp") + "#{Rails.root}/tmp/test.jpg" + end +end + +Factory.define(:uploaded_large_jpg_upload, :parent => :upload) do |f| + f.file_ext "jpg" + f.content_type "image/jpeg" + f.file_path do + FileUtils.cp("#{Rails.root}/test/files/test-large.jpg", "#{Rails.root}/tmp") + "#{Rails.root}/tmp/test-large.jpg" + end +end + +Factory.define(:uploaded_png_upload, :parent => :upload) do |f| + f.file_path do + FileUtils.cp("#{Rails.root}/test/files/test.png", "#{Rails.root}/tmp") + "#{Rails.root}/tmp/test.png" + end +end + +Factory.define(:uploaded_gif_upload, :parent => :upload) do |f| + f.file_path do + FileUtils.cp("#{Rails.root}/test/files/test.gif", "#{Rails.root}/tmp") + "#{Rails.root}/tmp/test.gif" + end +end diff --git a/test/unit/upload_test.rb b/test/unit/upload_test.rb new file mode 100644 index 000000000..b2b408dcd --- /dev/null +++ b/test/unit/upload_test.rb @@ -0,0 +1,156 @@ +require File.dirname(__FILE__) + '/../test_helper' + +class UploadTest < ActiveSupport::TestCase + context "An upload" do + teardown do + FileUtils.rm_f(Dir.glob("#{Rails.root}/tmp/test.*")) + end + + context "image size calculator" do + should "discover the dimensions for a JPG" do + @upload = Factory.create(:uploaded_jpg_upload) + assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} + assert_equal(500, @upload.image_width) + assert_equal(335, @upload.image_height) + end + + should "discover the dimensions for a PNG" do + @upload = Factory.create(:uploaded_png_upload) + assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} + assert_equal(768, @upload.image_width) + assert_equal(1024, @upload.image_height) + end + + should "discover the dimensions for a GIF" do + @upload = Factory.create(:uploaded_gif_upload) + assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)} + assert_equal(400, @upload.image_width) + assert_equal(400, @upload.image_height) + end + end + + context "content type calculator" do + should "know how to parse jpeg, png, gif, and swf file extensions" do + @upload = Factory.create(:uploaded_jpg_upload) + assert_equal("image/jpeg", @upload.file_ext_to_content_type("test.jpeg")) + assert_equal("image/gif", @upload.file_ext_to_content_type("test.gif")) + assert_equal("image/png", @upload.file_ext_to_content_type("test.png")) + assert_equal("application/x-shockwave-flash", @upload.file_ext_to_content_type("test.swf")) + assert_equal("application/octet-stream", @upload.file_ext_to_content_type("")) + end + + should "know how to parse jpeg, png, gif, and swf content types" do + @upload = Factory.create(:uploaded_jpg_upload) + assert_equal("jpg", @upload.content_type_to_file_ext("image/jpeg")) + assert_equal("gif", @upload.content_type_to_file_ext("image/gif")) + assert_equal("png", @upload.content_type_to_file_ext("image/png")) + assert_equal("swf", @upload.content_type_to_file_ext("application/x-shockwave-flash")) + assert_equal("bin", @upload.content_type_to_file_ext("")) + end + end + + context "downloader" do + should "initialize the final path and content type after downloading a file" do + @upload = Factory.create(:downloadable_upload) + path = "#{Rails.root}/tmp/test.download.jpg" + assert_nothing_raised {@upload.download_from_source(path)} + assert(File.exists?(path)) + assert_equal(8558, File.size(path)) + assert_equal("image/gif", @upload.content_type) + assert_equal(path, @upload.file_path) + assert_equal("gif", @upload.file_ext) + end + end + + context "file processor" do + should "parse and process a cgi file representation" do + FileUtils.cp("#{Rails.root}/test/files/test.jpg", "#{Rails.root}/tmp") + @upload = Upload.new(:file => upload_jpeg("#{Rails.root}/tmp/test.jpg")) + assert_nothing_raised {@upload.convert_cgi_file} + assert_equal("image/jpeg", @upload.content_type) + assert(File.exists?(@upload.file_path)) + assert_equal(28086, File.size(@upload.file_path)) + assert_equal("jpg", @upload.file_ext) + end + end + + context "hash calculator" do + should "caculate the hash" do + @upload = Factory.create(:uploaded_jpg_upload) + @upload.calculate_hash(@upload.file_path) + assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5) + end + end + + context "resizer" do + teardown do + FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/thumb/test.*.jpg")) + FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/medium/test.*.jpg")) + FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/large/test.*.jpg")) + FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/original/test.*.jpg")) + end + + should "generate several resized versions of the image" do + @upload = Factory.create(:uploaded_large_jpg_upload) + @upload.calculate_hash(@upload.file_path) + @upload.calculate_dimensions(@upload.file_path) + assert_nothing_raised {@upload.generate_resizes(@upload.file_path)} + assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.small_image_width))) + assert_equal(6556, File.size(@upload.resized_file_path_for(Danbooru.config.small_image_width))) + assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.medium_image_width))) + assert_equal(39411, File.size(@upload.resized_file_path_for(Danbooru.config.medium_image_width))) + assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.large_image_width))) + assert_equal(179324, File.size(@upload.resized_file_path_for(Danbooru.config.large_image_width))) + end + end + + should "process completely for a downloaded image" do + @upload = Factory.create(:downloadable_upload, + :rating => "s", + :uploader_ip_addr => "127.0.0.1", + :tag_string => "hoge foo" + ) + assert_difference("Post.count") do + assert_nothing_raised {@upload.process!} + end + + post = Post.last + assert_equal("hoge foo", post.tag_string) + assert_equal("s", post.rating) + assert_equal(@upload.uploader_id, post.uploader_id) + assert_equal("127.0.0.1", post.uploader_ip_addr) + assert_equal(@upload.md5, post.md5) + assert_equal("gif", post.file_ext) + assert_equal(276, post.image_width) + assert_equal(110, post.image_height) + assert_equal(8558, post.file_size) + assert_equal(post.id, @upload.post_id) + assert_equal("finished", @upload.status) + end + end + + should "process completely for an uploaded image" do + @upload = Factory.create(:uploaded_jpg_upload, + :rating => "s", + :uploader_ip_addr => "127.0.0.1", + :tag_string => "hoge foo" + ) + @upload.file = upload_jpeg("#{Rails.root}/test/files/test.jpg") + @upload.convert_cgi_file + + assert_difference("Post.count") do + assert_nothing_raised {@upload.process!} + end + post = Post.last + assert_equal("hoge foo", post.tag_string) + assert_equal("s", post.rating) + assert_equal(@upload.uploader_id, post.uploader_id) + assert_equal("127.0.0.1", post.uploader_ip_addr) + assert_equal(@upload.md5, post.md5) + assert_equal("jpg", post.file_ext) + assert(File.exists?(post.file_path)) + assert_equal(28086, File.size(post.file_path)) + assert_equal(post.id, @upload.post_id) + assert_equal("finished", @upload.status) + end +end