more work on post uploads

.gitignore

@@ -6,3 +6,4 @@ lib/danbooru_image_resizer/*.log
 db/*.sqlite3
 log/*.log
 tmp/**/*
+public/data

app/controllers/application_controller.rb

@@ -10,10 +10,10 @@ protected
 
     respond_to do |fmt|
       fmt.html do 
-        if request.get? && Rails.environment != "test"
+        if request.get? && Rails.env.test?
-          redirect_to new_sessions_path(:url => previous_url), :notice => "Access denied"
+          redirect_to new_session_path(:url => previous_url), :notice => "Access denied"
         else
-          redirect_to new_sessions_path, :notice => "Access denied"
+          redirect_to new_session_path, :notice => "Access denied"
         end
       end
       fmt.xml do
@@ -38,9 +38,11 @@ protected
     else
       @current_user = AnonymousUser.new
     end
+    
+    Time.zone = @current_user.time_zone
   end
   
-  %w(banned privileged contributor janitor moderator admin).each do |level|
+  %w(member banned privileged contributor janitor moderator admin).each do |level|
     define_method("#{level}_only") do
       if @current_user.__send__("is_#{level}?")
         true

app/controllers/uploads_controller.rb

@@ -1,54 +1,20 @@
 class UploadsController < ApplicationController
+  before_filter :member_only
+  respond_to :html, :xml, :json
+  
   def new
-    @upload = Upload.new
+    @upload = Upload.new(:rating => "q")
     if params[:url]
       @post = Post.find_by_source(params[:url])
     end
   end
   
   def show
+    @upload = Upload.find(params[:id])
   end
 
   def create
-    unless @current_user.can_upload?
+  	@upload = Upload.create(params[:upload].merge(:uploader_id => @current_user.id, :uploader_ip_addr => request.remote_ip))
-      respond_to_error("Daily limit exceeded", {:controller => "user", :action => "upload_limit"}, :status => 421)
+    respond_with(@upload)
-      return
-    end
-
-    if @current_user.is_contributor_or_higher?
-      status = "active"
-    else
-      status = "pending"
-    end
-
-		begin
-    	@post = Post.new(params[:post].merge(:updater_user_id => @current_user.id, :updater_ip_addr => request.remote_ip))
-    	@post.user_id = @current_user.id
-    	@post.status = status
-    	@post.ip_addr = request.remote_ip
-    	@post.save
-		rescue Errno::ENOENT
-			respond_to_error("Internal error. Try uploading again.", {:controller => "post", :action => "error"})
-			return
-		end
-
-    if @post.errors.empty?
-      if params[:md5] && @post.md5 != params[:md5].downcase
-        @post.destroy
-        respond_to_error("MD5 mismatch", {:action => "error"}, :status => 420)
-      else
-        respond_to_success("Post uploaded", {:controller => "post", :action => "show", :id => @post.id, :tag_title => @post.tag_title}, :api => {:post_id => @post.id, :location => url_for(:controller => "post", :action => "show", :id => @post.id)})
-      end
-    elsif @post.errors.invalid?(:md5)
-      p = Post.find_by_md5(@post.md5)
-
-      update = { :tags => p.cached_tags + " " + params[:post][:tags], :updater_user_id => session[:user_id], :updater_ip_addr => request.remote_ip }
-      update[:source] = @post.source if p.source.blank? && !@post.source.blank?
-      p.update_attributes(update)
-
-      respond_to_error("Post already exists", {:controller => "post", :action => "show", :id => p.id, :tag_title => @post.tag_title}, :api => {:location => url_for(:controller => "post", :action => "show", :id => p.id)}, :status => 423)
-    else
-      respond_to_error(@post, :action => "error")
-    end
   end
 end

app/logical/anonymous_user.rb

@@ -99,11 +99,13 @@ class AnonymousUser
   def blacklisted_tags
     ""
   end
-  
-  %w(banned privileged contributor janitor moderator admin).each do |name, value|
-    normalized_name = name.downcase.gsub(/ /, "_")
 
-    define_method("is_#{normalized_name}?") do
+  def time_zone
+    "Eastern Time (US & Canada)"
+  end
+  
+  %w(member banned privileged contributor janitor moderator admin).each do |name|
+    define_method("is_#{name}?") do
       false
     end
   end

app/models/job.rb

@@ -1,5 +1,5 @@
 class Job < ActiveRecord::Base
-  CATEGORIES = %w(mass_tag_edit approve_tag_alias approve_tag_implication calculate_tag_subscriptions calculate_related_tags calculate_post_count calculate_uploaded_tags s3_backup)
+  CATEGORIES = %w(mass_tag_edit approve_tag_alias approve_tag_implication calculate_tag_subscriptions calculate_related_tags s3_backup upload_processing)
   STATUSES = %w(pending processing finished error)
   
   validates_inclusion_of :category, :in => CATEGORIES
@@ -37,6 +37,12 @@ class Job < ActiveRecord::Base
     end
   end
   
+  def execute_upload_processing
+    Upload.where("status = ?", "pending").each do |upload|
+      upload.process!
+    end
+  end
+  
   def execute_mass_tag_edit
     start_tags = data["start_tags"]
     result_tags = data["result_tags"]
@@ -75,52 +81,32 @@ class Job < ActiveRecord::Base
     end
   end
   
-  def execute_calculate_post_count
-    Tag.recalculate_post_count(data["tag_name"])
-  end
-  
-  def execute_calculate_uploaded_tags
-    tags = []
-    user = User.find(data["id"])
-    CONFIG["tag_types"].values.uniq.each do |tag_type|
-      tags += user.calculate_uploaded_tags(tag_type)
-    end
-    
-    user.update_attribute(:uploaded_tags, tags.join("\n"))
-  end
-  
-  def execute_bandwidth_throttle
-    bw = File.read("/proc/net/dev").split(/\n/).grep(/eth1/).first.scan(/\S+/)[8].to_i
-    if $danbooru_bandwidth_previous
-      diff = bw - $danbooru_bandwidth_previous
-    else
-      diff = 0
-    end
-    $danbooru_bandwidth_previous = bw
-    Cache.put("db-bw", diff)
-  end
-  
   def execute_s3_backup
     last_id = data["last_id"].to_i
     
     begin
-      Post.find(:all, :conditions => ["id > ?", last_id], :limit => 200, :order => "id").each do |post|
+      Post.where("id > ?", last_id).each do |post|
-        AWS::S3::Base.establish_connection!(:access_key_id => CONFIG["amazon_s3_access_key_id"], :secret_access_key => CONFIG["amazon_s3_secret_access_key"])
+        AWS::S3::Base.establish_connection!(
+          :access_key_id => Danbooru.config.amazon_s3_access_key_id, 
+          :secret_access_key => Danbooru.config.amazon_s3_secret_access_key
+        )
         if File.exists?(post.file_path)
-          base64_md5 = Base64.encode64(Digest::MD5.digest(File.read(post.file_path)))
+          AWS::S3::S3Object.store(
-          AWS::S3::S3Object.store(post.file_name, open(post.file_path, "rb"), CONFIG["amazon_s3_bucket_name"], "Content-MD5" => base64_md5)
+            post.file_name, 
+            open(post.file_path, "rb"), 
+            Danbooru.config.amazon_s3_bucket_name, 
+            "Content-MD5" => Base64.encode64(post.md5)
+          )
         end
-
         if post.image? && File.exists?(post.preview_path)
-          AWS::S3::S3Object.store("preview/#{post.md5}.jpg", open(post.preview_path, "rb"), CONFIG["amazon_s3_bucket_name"])
+          AWS::S3::S3Object.store(
+            "preview/#{post.md5}.jpg", 
+            open(post.preview_path, "rb"), 
+            Danbooru.config.amazon_s3_bucket_name
+          )
         end
 
-        if File.exists?(post.sample_path)
-          AWS::S3::S3Object.store("sample/" + CONFIG["sample_filename_prefix"] + "#{post.md5}.jpg", open(post.sample_path, "rb"), CONFIG["amazon_s3_bucket_name"])
-        end
-      
         update_attributes(:data => {:last_id => post.id})
-        base64_md5 = nil
       end
 
     rescue Exception => x
@@ -156,19 +142,13 @@ class Job < ActiveRecord::Base
         else
           "tag:UNKNOWN"
         end
-    
+      
-      when "calculate_post_count"
-        "tag:" + data["tag_name"]
-        
-      when "calculate_uploaded_tags"
-        "user:" + User.name(data["id"])
-        
       when "bandwidth_throttle"
         ""
-        
+      
       when "s3_backup"
         "last_id:" + data["last_id"].to_s
-        
+      
       end
     rescue Exception
       "ERROR"
@@ -176,11 +156,11 @@ class Job < ActiveRecord::Base
   end
   
   def self.pending_count(task_type)
-    JobTask.count(:conditions => ["task_type = ? and status = 'pending'", task_type])
+    where("task_type = ? and status = 'pending'", task_type).count
   end
   
   def self.execute_once
-    find(:all, :conditions => ["status = ?", "pending"], :order => "id desc").each do |task|
+    where("status = ?", "pending").each do |task|
       task.execute!
       sleep 1
     end

app/models/post.rb

@@ -23,7 +23,7 @@ class Post < ActiveRecord::Base
       FileUtils.rm_f(file_path)
       FileUtils.rm_f(medium_file_path)
       FileUtils.rm_f(large_file_path)
-      FileUtils.rm_f(thumb_file_path)
+      FileUtils.rm_f(preview_file_path)
     end
 
     def file_path_prefix
@@ -42,8 +42,8 @@ class Post < ActiveRecord::Base
       "#{Rails.root}/public/data/large/#{file_path_prefix}#{md5}.jpg"
     end
 
-    def thumb_file_path
+    def preview_file_path
-      "#{Rails.root}/public/data/thumb/#{file_path_prefix}#{md5}.jpg"
+      "#{Rails.root}/public/data/preview/#{file_path_prefix}#{md5}.jpg"
     end
 
     def file_url
@@ -58,8 +58,8 @@ class Post < ActiveRecord::Base
       "/data/large/#{file_path_prefix}#{md5}.jpg"
     end
 
-    def thumb_file_url
+    def preview_file_url
-      "/data/thumb/#{file_path_prefix}#{md5}.jpg"
+      "/data/preview/#{file_path_prefix}#{md5}.jpg"
     end
     
     def file_url_for(user)

app/models/upload.rb

@@ -2,62 +2,106 @@ require "danbooru_image_resizer/danbooru_image_resizer"
 require "tmpdir"
 
 class Upload < ActiveRecord::Base
+  class Error < Exception ; end
+  
   attr_accessor :file, :image_width, :image_height, :file_ext, :md5, :file_size
   belongs_to :uploader, :class_name => "User"
   belongs_to :post
-  before_save :convert_cgi_file
+  before_create :initialize_status
+  before_create :convert_cgi_file
+  validate :uploader_is_not_limited
   
-  def process!
+  module ValidationMethods
-    update_attribute(:status, "processing")
+    def uploader_is_not_limited
-    if is_downloadable?
+      if !uploader.can_upload?
-      download_from_source(temp_file_path)
+        update_attribute(:status, "error: uploader has reached their daily limit")
+        raise
+      end
     end
-    self.file_ext = content_type_to_file_ext(content_type)
+    
-    calculate_hash(file_path)
+    def validate_file_exists
-    calculate_file_size(file_path)
+      unless File.exists?(file_path)
-    calculate_dimensions(file_path) if has_dimensions?
+        update_attribute(:status, "error: file does not exist")
-    generate_resizes(file_path)
+        raise
-    move_file
+      end
-    post = convert_to_post
+    end
-    if post.save
+    
-      update_attributes(:status => "finished", :post_id => post.id)
+    def validate_file_content_type
-    else
+      unless is_valid_content_type?
-      update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
+        update_attribute(:status, "error: invalid content type (#{file_ext} not allowed)")
+        raise
+      end
+    end
+    
+    def validate_md5_confirmation
+      if !md5_confirmation.blank? && md5_confirmation != md5
+        update_attribute(:status, "error: md5 mismatch")
+        raise
+      end
+    end
+  end
+  
+  module ConversionMethods
+    def process!
+      update_attribute(:status, "processing")
+      if is_downloadable?
+        download_from_source(temp_file_path)
+      end
+      validate_file_exists
+      self.file_ext = content_type_to_file_ext(content_type)
+      validate_file_content_type
+      calculate_hash(file_path)
+      validate_md5_confirmation
+      calculate_file_size(file_path)
+      calculate_dimensions(file_path) if has_dimensions?
+      generate_resizes(file_path)
+      move_file
+      post = convert_to_post
+      if post.save
+        update_attributes(:status => "completed", :post_id => post.id)
+      else
+        update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
+      end
+    rescue RuntimeError => x
+    end
+
+    def convert_to_post
+      returning Post.new do |p|
+        p.tag_string = tag_string
+        p.md5 = md5
+        p.file_ext = file_ext
+        p.image_width = image_width
+        p.image_height = image_height
+        p.uploader_id = uploader_id
+        p.uploader_ip_addr = uploader_ip_addr
+        p.updater_id = uploader_id
+        p.updater_ip_addr = uploader_ip_addr
+        p.rating = rating
+        p.source = source
+        p.file_size = file_size
+
+        unless uploader.is_contributor?
+          p.is_pending = true
+        end
+      end
+    end
+  end
+  
+  module FileMethods
+    def move_file
+      FileUtils.mv(file_path, md5_file_path)
+    end
+
+    def calculate_file_size(source_path)
+      self.file_size = File.size(source_path)
+    end
+
+    # Calculates the MD5 based on whatever is in temp_file_path
+    def calculate_hash(source_path)
+      self.md5 = Digest::MD5.file(source_path).hexdigest
     end
   end
 
-  def convert_to_post
-    returning Post.new do |p|
-      p.tag_string = tag_string
-      p.md5 = md5
-      p.file_ext = file_ext
-      p.image_width = image_width
-      p.image_height = image_height
-      p.uploader_id = uploader_id
-      p.uploader_ip_addr = uploader_ip_addr
-      p.updater_id = uploader_id
-      p.updater_ip_addr = uploader_ip_addr
-      p.rating = rating
-      p.source = source
-      p.file_size = file_size      
-    end
-  end
-  
-  def move_file
-    FileUtils.mv(file_path, md5_file_path)
-  end
-  
-  def calculate_file_size(source_path)
-    self.file_size = File.size(source_path)
-  end
-  
-  # Calculates the MD5 based on whatever is in temp_file_path
-  def calculate_hash(source_path)
-    self.md5 = Digest::MD5.file(source_path).hexdigest
-  end
-
-  class Error < Exception ; end
-  
   module ResizerMethods
     def generate_resizes(source_path)
       generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path)
@@ -101,6 +145,10 @@ class Upload < ActiveRecord::Base
   end
   
   module ContentTypeMethods
+    def is_valid_content_type?
+      file_ext =~ /jpg|gif|png|swf/
+    end
+    
     def content_type_to_file_ext(content_type)
       case content_type
       when "image/jpeg"
@@ -152,7 +200,7 @@ class Upload < ActiveRecord::Base
 
       case width
       when Danbooru.config.small_image_width
-        "#{Rails.root}/public/data/thumb/#{prefix}#{md5}.jpg"
+        "#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg"
 
       when Danbooru.config.medium_image_width
         "#{Rails.root}/public/data/medium/#{prefix}#{md5}.jpg"
@@ -202,10 +250,28 @@ class Upload < ActiveRecord::Base
     end
   end
   
+  module StatusMethods
+    def initialize_status
+      self.status = "pending"
+    end
+    
+    def is_pending?
+      status == "pending"
+    end
+    
+    def is_completed?
+      status == "completed"
+    end
+  end
+  
+  include ConversionMethods
+  include ValidationMethods
+  include FileMethods
   include ResizerMethods
   include DimensionMethods
   include ContentTypeMethods
   include DownloaderMethods
   include FilePathMethods
   include CgiFileMethods
+  include StatusMethods
 end

app/models/user.rb

@@ -135,6 +135,10 @@ class User < ActiveRecord::Base
     def is_anonymous?
       false
     end
+
+    def is_member?
+      true
+    end
   end
   
   module EmailVerificationMethods

app/presenters/post_set_presenter.rb

@@ -1,4 +1,4 @@
-class PostSetPresenter
+class PostSetPresenter < Presenter
   attr_accessor :post_set
   
   def initialize(post_set)
@@ -24,13 +24,18 @@ class PostSetPresenter
     html = ""
     
     posts.each do |post|
-      html << %{<article id="post_#{post.id}" data-id="#{post.id}" data-tags="#{h(post.tag_string)}" data-uploader="#{h(post.uploader_name)}" data-rating="#{post.rating}" data-width="#{post.image_width}" data-height="#{post.image_height}">}
+      flags = []
+      flags << "pending" if post.is_pending?
+      flags << "flagged" if post.is_flagged?
+      flags << "deleted" if post.is_deleted?
+      
+      html << %{<article id="post_#{post.id}" data-id="#{post.id}" data-tags="#{h(post.tag_string)}" data-uploader="#{h(post.uploader_name)}" data-rating="#{post.rating}" data-width="#{post.image_width}" data-height="#{post.image_height}" data-flags="#{flags.join(' ')}">}
       html << %{<a href="/posts/#{post.id}">}
-      html << %{<img src="#{post.preview_url}>"}
+      html << %{<img src="#{post.preview_file_url}">}
       html << %{</a>}
       html << %{</article>}
     end
     
-    html
+    html.html_safe
   end
 end

app/presenters/presenter.rb

@@ -0,0 +1,5 @@
+class Presenter
+  def h(s)
+    CGI.escapeHTML(s)
+  end
+end

app/views/sessions/new.html.erb

@@ -28,9 +28,5 @@
 </aside>
 
 <% content_for(:page_title) do %>
-login
+ / login
-<% end %>
-
-<% content_for(:page_header) do %>
-  / login
 <% end %>

app/views/uploads/new.html.erb

@@ -2,7 +2,7 @@
   <p>Before uploading, please read the <%= link_to "how to upload guide", wiki_page_path("howto:upload") %>. It explains how to tag and what ratings are.</p>
 </div>
 
-<% form_for @upload do |f| %>
+<% form_for(@upload, :html => {:multipart => true}) do |f| %>
   <% if params[:url] %>
     <div id="image-preview">
       <%= image_tag(params[:url], :title => "Preview") %>

app/views/uploads/show.html.erb

@@ -0,0 +1,15 @@
+<h1>Upload #<%= @upload.id %></h1>
+
+<ul>
+  <li>Date: <%= @upload.created_at %></li>
+  <li>Source: <%= @upload.source %></li>
+  <li>Tags: <%= @upload.tag_string %></li>
+</ul>
+
+<% if @upload.is_completed? %>
+  <p>This upload has finished processing. <%= link_to "View the post", post_path(@upload.post_id) %>.</p>
+<% elsif @upload.is_pending? %>
+  <p>This upload is waiting to be processed. Please wait a few seconds.</p>
+<% else %>
+  <p>An error occurred: <%= @upload.status %></p>
+<% end %>

app/views/users/edit.html.erb

@@ -13,6 +13,11 @@
       <% end %>
     </p>
     
+    <p>
+      <%= f.label :time_zone %>
+      <%= f.time_zone_select :time_zone %>
+    </p>
+    
     <p>
       <%= f.label :receive_email_notifications, "Email notifications", :title => "Enable to receive email notification when you receive a DMail" %>
       <%= f.check_box :receive_email_notifications %>

config/initializers/create_empty_directories.rb

@@ -1,6 +1,6 @@
 require 'fileutils'
 
-FileUtils.mkdir_p("#{Rails.root}/public/data/thumb")
+FileUtils.mkdir_p("#{Rails.root}/public/data/preview")
 FileUtils.mkdir_p("#{Rails.root}/public/data/medium")
 FileUtils.mkdir_p("#{Rails.root}/public/data/large")
 FileUtils.mkdir_p("#{Rails.root}/public/data/original")

db/development_structure.sql

@@ -1324,8 +1324,6 @@ ALTER SEQUENCE unapprovals_id_seq OWNED BY unapprovals.id;
 
 CREATE TABLE uploads (
     id integer NOT NULL,
-    created_at timestamp without time zone,
-    updated_at timestamp without time zone,
     source character varying(255),
     file_path character varying(255),
     content_type character varying(255),
@@ -1334,7 +1332,10 @@ CREATE TABLE uploads (
     uploader_ip_addr inet NOT NULL,
     tag_string text NOT NULL,
     status character varying(255) DEFAULT 'pending'::character varying NOT NULL,
-    post_id integer
+    post_id integer,
+    md5_confirmation character varying(255),
+    created_at timestamp without time zone,
+    updated_at timestamp without time zone
 );
 
 
@@ -1419,7 +1420,8 @@ CREATE TABLE users (
     always_resize_images boolean DEFAULT false NOT NULL,
     default_image_size character varying(255) DEFAULT 'medium'::character varying NOT NULL,
     favorite_tags text,
-    blacklisted_tags text
+    blacklisted_tags text,
+    time_zone character varying(255) DEFAULT 'Eastern Time (US & Canada)'::character varying NOT NULL
 );
 
 

db/migrate/20100204211522_create_users.rb

@@ -28,6 +28,7 @@ class CreateUsers < ActiveRecord::Migration
       t.column :default_image_size, :string, :null => false, :default => "medium"
       t.column :favorite_tags, :text
       t.column :blacklisted_tags, :text
+      t.column :time_zone, :string, :null => false, :default => "Eastern Time (US & Canada)"
     end
     
     execute "CREATE UNIQUE INDEX index_users_on_name ON users ((lower(name)))"

db/migrate/20100205224030_create_uploads.rb

@@ -1,17 +1,17 @@
 class CreateUploads < ActiveRecord::Migration
   def self.up
     create_table :uploads do |t|
-      t.timestamps
       t.column :source, :string
       t.column :file_path, :string
       t.column :content_type, :string
-      
       t.column :rating, :character, :null => false
       t.column :uploader_id, :integer, :null => false
       t.column :uploader_ip_addr, "inet", :null => false
       t.column :tag_string, :text, :null => false
       t.column :status, :string, :null => false, :default => "pending"
       t.column :post_id, :integer
+      t.column :md5_confirmation, :string
+      t.timestamps
     end
   end