more work on post uploads

2010-03-12 19:27:54 -05:00
parent 9eb578927c
commit ca8be10ab9
18 changed files with 218 additions and 168 deletions
--- a/app/models/job.rb
+++ b/app/models/job.rb
@@ -1,5 +1,5 @@
 class Job < ActiveRecord::Base
-  CATEGORIES = %w(mass_tag_edit approve_tag_alias approve_tag_implication calculate_tag_subscriptions calculate_related_tags calculate_post_count calculate_uploaded_tags s3_backup)
+  CATEGORIES = %w(mass_tag_edit approve_tag_alias approve_tag_implication calculate_tag_subscriptions calculate_related_tags s3_backup upload_processing)
  STATUSES = %w(pending processing finished error)
  
  validates_inclusion_of :category, :in => CATEGORIES
@@ -37,6 +37,12 @@ class Job < ActiveRecord::Base
    end
  end
  
+  def execute_upload_processing
+    Upload.where("status = ?", "pending").each do |upload|
+      upload.process!
+    end
+  end
+  
  def execute_mass_tag_edit
    start_tags = data["start_tags"]
    result_tags = data["result_tags"]
@@ -75,52 +81,32 @@ class Job < ActiveRecord::Base
    end
  end
  
-  def execute_calculate_post_count
-    Tag.recalculate_post_count(data["tag_name"])
-  end
-  
-  def execute_calculate_uploaded_tags
-    tags = []
-    user = User.find(data["id"])
-    CONFIG["tag_types"].values.uniq.each do |tag_type|
-      tags += user.calculate_uploaded_tags(tag_type)
-    end
-    
-    user.update_attribute(:uploaded_tags, tags.join("\n"))
-  end
-  
-  def execute_bandwidth_throttle
-    bw = File.read("/proc/net/dev").split(/\n/).grep(/eth1/).first.scan(/\S+/)[8].to_i
-    if $danbooru_bandwidth_previous
-      diff = bw - $danbooru_bandwidth_previous
-    else
-      diff = 0
-    end
-    $danbooru_bandwidth_previous = bw
-    Cache.put("db-bw", diff)
-  end
-  
  def execute_s3_backup
    last_id = data["last_id"].to_i
    
    begin
-      Post.find(:all, :conditions => ["id > ?", last_id], :limit => 200, :order => "id").each do |post|
-        AWS::S3::Base.establish_connection!(:access_key_id => CONFIG["amazon_s3_access_key_id"], :secret_access_key => CONFIG["amazon_s3_secret_access_key"])
+      Post.where("id > ?", last_id).each do |post|
+        AWS::S3::Base.establish_connection!(
+          :access_key_id => Danbooru.config.amazon_s3_access_key_id, 
+          :secret_access_key => Danbooru.config.amazon_s3_secret_access_key
+        )
        if File.exists?(post.file_path)
-          base64_md5 = Base64.encode64(Digest::MD5.digest(File.read(post.file_path)))
-          AWS::S3::S3Object.store(post.file_name, open(post.file_path, "rb"), CONFIG["amazon_s3_bucket_name"], "Content-MD5" => base64_md5)
+          AWS::S3::S3Object.store(
+            post.file_name, 
+            open(post.file_path, "rb"), 
+            Danbooru.config.amazon_s3_bucket_name, 
+            "Content-MD5" => Base64.encode64(post.md5)
+          )
        end
-
        if post.image? && File.exists?(post.preview_path)
-          AWS::S3::S3Object.store("preview/#{post.md5}.jpg", open(post.preview_path, "rb"), CONFIG["amazon_s3_bucket_name"])
+          AWS::S3::S3Object.store(
+            "preview/#{post.md5}.jpg", 
+            open(post.preview_path, "rb"), 
+            Danbooru.config.amazon_s3_bucket_name
+          )
        end

-        if File.exists?(post.sample_path)
-          AWS::S3::S3Object.store("sample/" + CONFIG["sample_filename_prefix"] + "#{post.md5}.jpg", open(post.sample_path, "rb"), CONFIG["amazon_s3_bucket_name"])
-        end
-      
        update_attributes(:data => {:last_id => post.id})
-        base64_md5 = nil
      end

    rescue Exception => x
@@ -156,19 +142,13 @@ class Job < ActiveRecord::Base
        else
          "tag:UNKNOWN"
        end
-    
-      when "calculate_post_count"
-        "tag:" + data["tag_name"]
-        
-      when "calculate_uploaded_tags"
-        "user:" + User.name(data["id"])
-        
+      
      when "bandwidth_throttle"
        ""
-        
+      
      when "s3_backup"
        "last_id:" + data["last_id"].to_s
-        
+      
      end
    rescue Exception
      "ERROR"
@@ -176,11 +156,11 @@ class Job < ActiveRecord::Base
  end
  
  def self.pending_count(task_type)
-    JobTask.count(:conditions => ["task_type = ? and status = 'pending'", task_type])
+    where("task_type = ? and status = 'pending'", task_type).count
  end
  
  def self.execute_once
-    find(:all, :conditions => ["status = ?", "pending"], :order => "id desc").each do |task|
+    where("status = ?", "pending").each do |task|
      task.execute!
      sleep 1
    end
--- a/app/models/post.rb
+++ b/app/models/post.rb
@@ -23,7 +23,7 @@ class Post < ActiveRecord::Base
      FileUtils.rm_f(file_path)
      FileUtils.rm_f(medium_file_path)
      FileUtils.rm_f(large_file_path)
-      FileUtils.rm_f(thumb_file_path)
+      FileUtils.rm_f(preview_file_path)
    end

    def file_path_prefix
@@ -42,8 +42,8 @@ class Post < ActiveRecord::Base
      "#{Rails.root}/public/data/large/#{file_path_prefix}#{md5}.jpg"
    end

-    def thumb_file_path
-      "#{Rails.root}/public/data/thumb/#{file_path_prefix}#{md5}.jpg"
+    def preview_file_path
+      "#{Rails.root}/public/data/preview/#{file_path_prefix}#{md5}.jpg"
    end

    def file_url
@@ -58,8 +58,8 @@ class Post < ActiveRecord::Base
      "/data/large/#{file_path_prefix}#{md5}.jpg"
    end

-    def thumb_file_url
-      "/data/thumb/#{file_path_prefix}#{md5}.jpg"
+    def preview_file_url
+      "/data/preview/#{file_path_prefix}#{md5}.jpg"
    end
    
    def file_url_for(user)
--- a/app/models/upload.rb
+++ b/app/models/upload.rb
@@ -2,62 +2,106 @@ require "danbooru_image_resizer/danbooru_image_resizer"
 require "tmpdir"

 class Upload < ActiveRecord::Base
+  class Error < Exception ; end
+  
  attr_accessor :file, :image_width, :image_height, :file_ext, :md5, :file_size
  belongs_to :uploader, :class_name => "User"
  belongs_to :post
-  before_save :convert_cgi_file
+  before_create :initialize_status
+  before_create :convert_cgi_file
+  validate :uploader_is_not_limited
  
-  def process!
-    update_attribute(:status, "processing")
-    if is_downloadable?
-      download_from_source(temp_file_path)
+  module ValidationMethods
+    def uploader_is_not_limited
+      if !uploader.can_upload?
+        update_attribute(:status, "error: uploader has reached their daily limit")
+        raise
+      end
    end
-    self.file_ext = content_type_to_file_ext(content_type)
-    calculate_hash(file_path)
-    calculate_file_size(file_path)
-    calculate_dimensions(file_path) if has_dimensions?
-    generate_resizes(file_path)
-    move_file
-    post = convert_to_post
-    if post.save
-      update_attributes(:status => "finished", :post_id => post.id)
-    else
-      update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
+    
+    def validate_file_exists
+      unless File.exists?(file_path)
+        update_attribute(:status, "error: file does not exist")
+        raise
+      end
+    end
+    
+    def validate_file_content_type
+      unless is_valid_content_type?
+        update_attribute(:status, "error: invalid content type (#{file_ext} not allowed)")
+        raise
+      end
+    end
+    
+    def validate_md5_confirmation
+      if !md5_confirmation.blank? && md5_confirmation != md5
+        update_attribute(:status, "error: md5 mismatch")
+        raise
+      end
+    end
+  end
+  
+  module ConversionMethods
+    def process!
+      update_attribute(:status, "processing")
+      if is_downloadable?
+        download_from_source(temp_file_path)
+      end
+      validate_file_exists
+      self.file_ext = content_type_to_file_ext(content_type)
+      validate_file_content_type
+      calculate_hash(file_path)
+      validate_md5_confirmation
+      calculate_file_size(file_path)
+      calculate_dimensions(file_path) if has_dimensions?
+      generate_resizes(file_path)
+      move_file
+      post = convert_to_post
+      if post.save
+        update_attributes(:status => "completed", :post_id => post.id)
+      else
+        update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
+      end
+    rescue RuntimeError => x
+    end
+
+    def convert_to_post
+      returning Post.new do |p|
+        p.tag_string = tag_string
+        p.md5 = md5
+        p.file_ext = file_ext
+        p.image_width = image_width
+        p.image_height = image_height
+        p.uploader_id = uploader_id
+        p.uploader_ip_addr = uploader_ip_addr
+        p.updater_id = uploader_id
+        p.updater_ip_addr = uploader_ip_addr
+        p.rating = rating
+        p.source = source
+        p.file_size = file_size
+
+        unless uploader.is_contributor?
+          p.is_pending = true
+        end
+      end
+    end
+  end
+  
+  module FileMethods
+    def move_file
+      FileUtils.mv(file_path, md5_file_path)
+    end
+
+    def calculate_file_size(source_path)
+      self.file_size = File.size(source_path)
+    end
+
+    # Calculates the MD5 based on whatever is in temp_file_path
+    def calculate_hash(source_path)
+      self.md5 = Digest::MD5.file(source_path).hexdigest
    end
  end

-  def convert_to_post
-    returning Post.new do |p|
-      p.tag_string = tag_string
-      p.md5 = md5
-      p.file_ext = file_ext
-      p.image_width = image_width
-      p.image_height = image_height
-      p.uploader_id = uploader_id
-      p.uploader_ip_addr = uploader_ip_addr
-      p.updater_id = uploader_id
-      p.updater_ip_addr = uploader_ip_addr
-      p.rating = rating
-      p.source = source
-      p.file_size = file_size      
-    end
-  end
-  
-  def move_file
-    FileUtils.mv(file_path, md5_file_path)
-  end
-  
-  def calculate_file_size(source_path)
-    self.file_size = File.size(source_path)
-  end
-  
-  # Calculates the MD5 based on whatever is in temp_file_path
-  def calculate_hash(source_path)
-    self.md5 = Digest::MD5.file(source_path).hexdigest
-  end
-
-  class Error < Exception ; end
-  
  module ResizerMethods
    def generate_resizes(source_path)
      generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path)
@@ -101,6 +145,10 @@ class Upload < ActiveRecord::Base
  end
  
  module ContentTypeMethods
+    def is_valid_content_type?
+      file_ext =~ /jpg|gif|png|swf/
+    end
+    
    def content_type_to_file_ext(content_type)
      case content_type
      when "image/jpeg"
@@ -152,7 +200,7 @@ class Upload < ActiveRecord::Base

      case width
      when Danbooru.config.small_image_width
-        "#{Rails.root}/public/data/thumb/#{prefix}#{md5}.jpg"
+        "#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg"

      when Danbooru.config.medium_image_width
        "#{Rails.root}/public/data/medium/#{prefix}#{md5}.jpg"
@@ -202,10 +250,28 @@ class Upload < ActiveRecord::Base
    end
  end
  
+  module StatusMethods
+    def initialize_status
+      self.status = "pending"
+    end
+    
+    def is_pending?
+      status == "pending"
+    end
+    
+    def is_completed?
+      status == "completed"
+    end
+  end
+  
+  include ConversionMethods
+  include ValidationMethods
+  include FileMethods
  include ResizerMethods
  include DimensionMethods
  include ContentTypeMethods
  include DownloaderMethods
  include FilePathMethods
  include CgiFileMethods
+  include StatusMethods
 end
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -135,6 +135,10 @@ class User < ActiveRecord::Base
    def is_anonymous?
      false
    end
+
+    def is_member?
+      true
+    end
  end
  
  module EmailVerificationMethods