This commit is contained in:
albert
2010-08-27 16:59:59 -04:00
parent ad39553aac
commit 6bc469b05d
34 changed files with 299 additions and 305 deletions

View File

@@ -12,3 +12,4 @@ gem "rails", "3.0.0.rc2"
gem "pg"
gem "memcache-client", :require => "memcache"
gem "imagesize", :require => "image_size"
gem "delayed_job"

View File

@@ -31,6 +31,9 @@ GEM
arel (1.0.0.rc1)
activesupport (>= 3.0.0.beta)
builder (2.1.2)
daemons (1.1.0)
delayed_job (2.0.3)
daemons
erubis (2.6.6)
abstract (>= 1.0.0)
factory_girl (1.3.1)
@@ -76,6 +79,7 @@ PLATFORMS
ruby
DEPENDENCIES
delayed_job
factory_girl
faker
imagesize

View File

@@ -1,16 +0,0 @@
class JobsController < ApplicationController
def edit
end
def index
end
def show
end
def destroy
end
def update
end
end

View File

@@ -17,14 +17,14 @@ class PostsController < ApplicationController
def update
@post = Post.find(params[:id])
@post.update_attributes(params[:post].merge(:updater_id => @current_user.id, :updater_ip_addr => request.remote_ip))
@post.update_attributes(params[:post])
respond_with(@post)
end
def revert
@post = Post.find(params[:id])
@version = PostVersion.find(params[:version_id])
@post.revert_to!(@version, @current_user.id, request.remote_ip)
@post.revert_to!(@version)
respond_width(@post)
end

View File

@@ -1,2 +0,0 @@
module JobsHelper
end

View File

@@ -1,4 +1,19 @@
class CurrentUser
def self.scoped(user, ip_addr)
old_user = self.user
old_ip_addr = self.ip_addr
self.user = user
self.ip_addr = ip_addr
begin
yield
ensure
self.user = old_user
self.ip_addr = old_ip_addr
end
end
def self.user=(user)
Thread.current[:current_user] = user
end

View File

@@ -7,7 +7,7 @@ class Download
@source = source
@file_path = file_path
end
# Downloads to @file_path
def download!
http_get_streaming(@source) do |response|
@@ -18,8 +18,8 @@ class Download
end
@source = fix_image_board_sources(@source)
end
# private
# private
def handle_pixiv(source, headers)
if source =~ /pixiv\.net/
headers["Referer"] = "http://www.pixiv.net"
@@ -33,7 +33,7 @@ class Download
source
end
def http_get_streaming(source, options = {})
max_size = options[:max_size] || Danbooru.config.max_file_size
max_size = nil if max_size == 0 # unlimited
@@ -77,7 +77,7 @@ class Download
end # http.start
end # while
end # def
def fix_image_board_sources(source)
if source =~ /\/src\/\d{12,}|urnc\.yi\.org|yui\.cynthia\.bne\.jp/
"Image board"

View File

@@ -1,13 +1,12 @@
class PostSet
class Error < Exception ; end
attr_accessor :tags, :page, :current_user, :before_id, :errors, :count
attr_accessor :tags, :page, :before_id, :errors, :count
attr_accessor :wiki_page, :artist, :posts, :suggestions
def initialize(tags, page, current_user, before_id = nil)
def initialize(tags, page, before_id = nil)
@tags = Tag.normalize(tags)
@page = page.to_i
@current_user = current_user
@before_id = before_id
@errors = []
load_associations
@@ -60,16 +59,14 @@ class PostSet
end
def tag_array
@tag_arary ||= Tag.scan_query(tags)
@tag_array ||= Tag.scan_query(tags)
end
def validate
begin
validate_page
validate_query_count
rescue Error => x
@errors << x.to_s
end
validate_page
validate_query_count
rescue Error => x
@errors << x.to_s
end
def validate_page
@@ -79,7 +76,7 @@ class PostSet
end
def validate_query_count
if !current_user.is_privileged? && tag_array.size > 2
if !CurrentUser.user.is_privileged? && tag_array.size > 2
raise Error.new("You can only search up to two tags at once with a basic account")
end

View File

@@ -1,3 +1,16 @@
class JanitorTrial < ActiveRecord::Base
belongs_to :user
after_create :send_dmail
def send_dmail
body = "You have been selected as a test janitor. You can now approve pending posts and have access to the moderation interface.\n\nOver the next several weeks your approvals will be monitored. If the majority of them are quality uploads, then you will be promoted to full janitor status which grants you the ability to delete and undelete posts, ban users, and revert tag changes from vandals. If you fail the trial period, you will be demoted back to your original level and you'll receive a negative user record indicating you previously attempted and failed a test janitor trial.\n\nThere is a minimum quota of 5 approvals a week to indicate that you are being active. Remember, the goal isn't to approve as much as possible. It's to filter out borderline-quality art.\n\nIf you have any questions please respond to this message."
dmail = Dmail.new(
:title => "Test Janitor Trial Period",
:body => body
)
dmail.from_id = User.admins.first.id
dmail.to_id = user_id
Dmail.create_new(dmail)
end
end

View File

@@ -1,168 +0,0 @@
class Job < ActiveRecord::Base
CATEGORIES = %w(mass_tag_edit approve_tag_alias approve_tag_implication calculate_tag_subscriptions calculate_related_tags s3_backup upload_processing)
STATUSES = %w(pending processing finished error)
validates_inclusion_of :category, :in => CATEGORIES
validates_inclusion_of :status, :in => STATUSES
def data
JSON.parse(data_as_json)
end
def data=(text)
self.data_as_json = text.to_json
end
def execute!
if repeat_count > 0
count = repeat_count - 1
else
count = repeat_count
end
begin
execute_sql("SET statement_timeout = 0")
update_attribute(:status, "processing")
__send__("execute_#{task_type}")
if count == 0
update_attribute(:status, "finished")
else
update_attributes(:status => "pending", :repeat_count => count)
end
rescue SystemExit => x
update_attribute(:status, "pending")
rescue Exception => x
update_attributes(:status => "error", :status_message => "#{x.class}: #{x}")
end
end
def execute_upload_processing
Upload.where("status = ?", "pending").each do |upload|
upload.process!
end
end
def execute_mass_tag_edit
start_tags = data["start_tags"]
result_tags = data["result_tags"]
updater_id = data["updater_id"]
updater_ip_addr = data["updater_ip_addr"]
Tag.mass_edit(start_tags, result_tags, updater_id, updater_ip_addr)
end
def execute_approve_tag_alias
ta = TagAlias.find(data["id"])
updater_id = data["updater_id"]
updater_ip_addr = data["updater_ip_addr"]
ta.approve(updater_id, updater_ip_addr)
end
def execute_approve_tag_implication
ti = TagImplication.find(data["id"])
updater_id = data["updater_id"]
updater_ip_addr = data["updater_ip_addr"]
ti.approve(updater_id, updater_ip_addr)
end
def execute_calculate_tag_subscriptions
last_run = Time.parse(data["last_run"])
if last_run.nil? || last_run < 20.minutes.ago
TagSubscription.process_all
update_attributes(:data => {:last_run => Time.now.strftime("%Y-%m-%d %H:%M")})
end
end
def execute_calculate_related_tags
tag_id = data["id"].to_i
tag = Tag.find_by_id(tag_id)
if tag
tag.commit_related(Tag.calculate_related(tag.name))
end
end
def execute_s3_backup
last_id = data["last_id"].to_i
begin
Post.where("id > ?", last_id).each do |post|
AWS::S3::Base.establish_connection!(
:access_key_id => Danbooru.config.amazon_s3_access_key_id,
:secret_access_key => Danbooru.config.amazon_s3_secret_access_key
)
if File.exists?(post.file_path)
AWS::S3::S3Object.store(
post.file_name,
open(post.file_path, "rb"),
Danbooru.config.amazon_s3_bucket_name,
"Content-MD5" => Base64.encode64(post.md5)
)
end
if post.image? && File.exists?(post.preview_path)
AWS::S3::S3Object.store(
"preview/#{post.md5}.jpg",
open(post.preview_path, "rb"),
Danbooru.config.amazon_s3_bucket_name
)
end
update_attributes(:data => {:last_id => post.id})
end
rescue Exception => x
# probably some network error, retry next time
end
end
def pretty_data
begin
case task_type
when "mass_tag_edit"
start = data["start_tags"]
result = data["result_tags"]
user = User.id_to_name(data["updater_id"])
"start:#{start} result:#{result} user:#{user}"
when "approve_tag_alias"
ta = TagAlias.find(data["id"])
"start:#{ta.name} result:#{ta.alias_name}"
when "approve_tag_implication"
ti = TagImplication.find(data["id"])
"start:#{ti.predicate.name} result:#{ti.consequent.name}"
when "calculate_tag_subscriptions"
last_run = data["last_run"]
"last run:#{last_run}"
when "calculate_related_tags"
tag = Tag.find_by_id(data["id"])
if tag
"tag:#{tag.name}"
else
"tag:UNKNOWN"
end
when "bandwidth_throttle"
""
when "s3_backup"
"last_id:" + data["last_id"].to_s
end
rescue Exception
"ERROR"
end
end
def self.pending_count(task_type)
where("task_type = ? and status = 'pending'", task_type).count
end
def self.execute_once
where("status = ?", "pending").each do |task|
task.execute!
sleep 1
end
end
end

View File

@@ -0,0 +1,27 @@
module Jobs
class BackupToS3 < Struct.new(:last_id)
def perform
Post.find(:all, :conditions => ["id > ?", last_id], :limit => 200, :order => "id").each do |post|
AWS::S3::Base.establish_connection!(:access_key_id => CONFIG["amazon_s3_access_key_id"], :secret_access_key => CONFIG["amazon_s3_secret_access_key"])
if File.exists?(post.file_path)
base64_md5 = Base64.encode64(Digest::MD5.digest(File.read(post.file_path)))
AWS::S3::S3Object.store(post.file_name, open(post.file_path, "rb"), CONFIG["amazon_s3_bucket_name"], "Content-MD5" => base64_md5)
end
if post.image? && File.exists?(post.preview_path)
AWS::S3::S3Object.store("preview/#{post.md5}.jpg", open(post.preview_path, "rb"), CONFIG["amazon_s3_bucket_name"])
end
if File.exists?(post.sample_path)
AWS::S3::S3Object.store("sample/" + CONFIG["sample_filename_prefix"] + "#{post.md5}.jpg", open(post.sample_path, "rb"), CONFIG["amazon_s3_bucket_name"])
end
self.last_id = post.id
end
Delayed::Job.enqueue(BackupToS3.new(last_id))
rescue Exception => x
# probably some network error, retry next time
end
end
end

View File

@@ -0,0 +1,7 @@
module Jobs
class CalculatePostCount < Struct.new(:tag_name)
def perform
Tag.recalculate_post_count(tag_name)
end
end
end

View File

@@ -0,0 +1,12 @@
module Jobs
class CalculateRelatedTags < Struct.new(:tag_id)
def perform
tag = Tag.find_by_id(tag_id)
if tag
tag.update_related
tag.save
end
end
end
end

View File

@@ -0,0 +1,13 @@
module Jobs
class CalculateUploadedTags < Struct.new(:user_id)
def perform
tags = []
user = User.find(user_id)
CONFIG["tag_types"].values.uniq.each do |tag_type|
tags += user.calculate_uploaded_tags(tag_type)
end
user.update_attribute(:uploaded_tags, tags.join("\n"))
end
end
end

View File

@@ -0,0 +1,12 @@
module Jobs
class CreateTagAlias < Struct.new(:antecedent_name, :consequent_name, :creator_id, :creator_ip_addr)
def execute
TagAlias.create(
:antecedent_name => antecedent_name,
:consequent_name => consequent_name,
:creator_id => creator_id,
:creator_ip_addr => creator_ip_addr
)
end
end
end

View File

@@ -0,0 +1,12 @@
module Jobs
class CreateTagImplication < Struct.new(:antecedent_name, :consequent_name, :creator_id, :creator_ip_addr)
def perform
TagImplication.create(
:antecedent_name => antecedent_name,
:consequent_name => consequent_name,
:creator_id => creator_id,
:creator_ip_addr => creator_ip_addr
)
end
end
end

View File

@@ -0,0 +1,13 @@
module Jobs
class FixPixivUploads < Struct.new(:last_post_id)
def perform
post_id = nil
Post.find_each(:conditions => ["GREATEST(width, height) IN (150, 600) AND source LIKE ? AND id > ?", "%pixiv%", last_post_id]) do |post|
post_id = post.id
end
update_attributes(:data => {:last_post_id => post_id})
end
end
end

View File

@@ -0,0 +1,7 @@
module Jobs
class MassTagEdit < Struct.new(:start_tags, :result_tags, :updater_id, :updater_ip_addr)
def perform
Tag.mass_edit(start_tags, result_tags, updater_id, updater_ip_addr)
end
end
end

View File

@@ -0,0 +1,10 @@
module Jobs
class ProcessTagSubscriptions < Struct.new(:last_run)
def perform
if last_run.nil? || last_run < 20.minutes.ago
TagSubscription.process_all
Delayed::Job.enqueue(ProcessTagSubscriptions.new(Time.now))
end
end
end
end

View File

@@ -0,0 +1,9 @@
module Jobs
class ProcessUploads
def perform
Upload.find_each(:conditions => ["status = ?", "pending"]) do |upload|
upload.process!
end
end
end
end

View File

@@ -108,13 +108,14 @@ class Tag < ActiveRecord::Base
module UpdateMethods
def mass_edit(start_tags, result_tags, updater_id, updater_ip_addr)
raise NotImplementedError
updater = User.find(updater_id)
Post.find_by_tags(start_tags).each do |p|
start = TagAlias.to_aliased(scan_tags(start_tags))
result = TagAlias.to_aliased(scan_tags(result_tags))
tags = (p.cached_tags.scan(/\S+/) - start + result).join(" ")
p.update_attributes(:updater_user_id => updater_id, :updater_ip_addr => updater_ip_addr, :tags => tags)
tags = (p.tag_array - start + result).join(" ")
CurrentUser.scoped(updater, updater_ip_addr) do
p.update_attributes(:tag_string => tags)
end
end
end
end

View File

@@ -1,11 +1,10 @@
class TagAlias < ActiveRecord::Base
attr_accessor :updater_id, :updater_ip_addr
attr_accessor :creator_ip_addr
after_save :update_posts
after_destroy :clear_cache
validates_presence_of :updater_id, :updater_ip_addr
validates_presence_of :creator_id, :creator_ip_addr
validates_uniqueness_of :antecedent_name
validate :absence_of_transitive_relation
belongs_to :updater, :class_name => "User"
belongs_to :creator, :class_name => "User"
def self.to_aliased(names)
@@ -41,11 +40,12 @@ class TagAlias < ActiveRecord::Base
Post.find_by_tags(antecedent_name).find_each do |post|
escaped_antecedent_name = Regexp.escape(antecedent_name)
fixed_tags = post.tag_string.sub(/(?:\A| )#{escaped_antecedent_name}(?:\Z| )/, " #{consequent_name} ").strip
post.update_attributes(
:tag_string => fixed_tags,
:updater_id => updater_id,
:updater_ip_addr => updater_ip_addr
)
CurrentUser.scoped(creator, creator_ip_addr) do
post.update_attributes(
:tag_string => fixed_tags
)
end
end
end
end

View File

@@ -1,13 +1,12 @@
class TagImplication < ActiveRecord::Base
attr_accessor :updater_id, :updater_ip_addr
attr_accessor :creator_ip_addr
before_save :clear_cache
before_save :update_descendant_names
after_save :update_descendant_names_for_parent
after_destroy :clear_cache
after_save :update_posts
belongs_to :creator, :class_name => "User"
belongs_to :updater, :class_name => "User"
validates_presence_of :updater_id, :updater_ip_addr, :creator_id
validates_presence_of :creator_id, :creator_ip_addr
validates_uniqueness_of :antecedent_name, :scope => :consequent_name
validate :absence_of_circular_relation
@@ -30,15 +29,16 @@ class TagImplication < ActiveRecord::Base
end
def descendants
all = []
children = [consequent_name]
@descendants ||= begin
[].tap do |all|
children = [consequent_name]
until children.empty?
all += children
children = self.class.where(["antecedent_name IN (?)", children]).all.map(&:consequent_name)
until children.empty?
all += children
children = self.class.where(["antecedent_name IN (?)", children]).all.map(&:consequent_name)
end
end
end
all
end
def descendant_names_array
@@ -51,16 +51,14 @@ class TagImplication < ActiveRecord::Base
self.descendant_names = descendants.join(" ")
end
def update_descendant_names!(updater_id, updater_ip_addr)
def update_descendant_names!
update_descendant_names
self.updater_id = updater_id
self.updater_ip_addr = updater_ip_addr
save!
end
def update_descendant_names_for_parent
if parent
parent.update_descendant_names!(updater_id, updater_ip_addr)
parent.update_descendant_names!
end
end
@@ -72,17 +70,22 @@ class TagImplication < ActiveRecord::Base
Post.find_by_tags(antecedent_name).find_each do |post|
escaped_antecedent_name = Regexp.escape(antecedent_name)
fixed_tags = post.tag_string.sub(/(?:\A| )#{escaped_antecedent_name}(?:\Z| )/, " #{antecedent_name} #{descendant_names} ").strip
post.update_attributes(
:tag_string => fixed_tags,
:updater_id => updater_id,
:updater_ip_addr => updater_ip_addr
)
CurrentUser.scoped(creator, creator_ip_addr) do
post.update_attributes(
:tag_string => fixed_tags
)
end
end
end
def reload(options = {})
super
clear_parent_cache
clear_descendants_cache
end
def clear_descendants_cache
@descendants = nil
end
def clear_parent_cache

View File

@@ -49,33 +49,28 @@ class Upload < ActiveRecord::Base
module ConversionMethods
def process!
CurrentUser.user = uploader
CurrentUser.ip_addr = uploader_ip_addr
update_attribute(:status, "processing")
if is_downloadable?
download_from_source(temp_file_path)
CurrentUser.scoped(uploader, uploader_ip_addr) do
update_attribute(:status, "processing")
if is_downloadable?
download_from_source(temp_file_path)
end
validate_file_exists
self.file_ext = content_type_to_file_ext(content_type)
validate_file_content_type
calculate_hash(file_path)
validate_md5_uniqueness
validate_md5_confirmation
calculate_file_size(file_path)
calculate_dimensions(file_path) if has_dimensions?
generate_resizes(file_path)
move_file
post = convert_to_post
if post.save
update_attributes(:status => "completed", :post_id => post.id)
else
update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
end
end
validate_file_exists
self.file_ext = content_type_to_file_ext(content_type)
validate_file_content_type
calculate_hash(file_path)
validate_md5_uniqueness
validate_md5_confirmation
calculate_file_size(file_path)
calculate_dimensions(file_path) if has_dimensions?
generate_resizes(file_path)
move_file
post = convert_to_post
if post.save
update_attributes(:status => "completed", :post_id => post.id)
else
update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
end
rescue RuntimeError => x
ensure
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
def convert_to_post
@@ -85,8 +80,6 @@ class Upload < ActiveRecord::Base
p.file_ext = file_ext
p.image_width = image_width
p.image_height = image_height
p.uploader_id = uploader_id
p.uploader_ip_addr = uploader_ip_addr
p.rating = rating
p.source = source
p.file_size = file_size

View File

@@ -22,7 +22,8 @@ class User < ActiveRecord::Base
has_many :feedback, :class_name => "UserFeedback", :dependent => :destroy
has_one :ban
belongs_to :inviter, :class_name => "User"
scope :named, lambda {|name| where(["lower(name) = ?", name])}
scope :named, lambda {|name| where(["lower(name) = ?", name])}
scope :admins, where("is_admin = TRUE")
module BanMethods
def validate_ip_addr_is_not_banned

View File

@@ -4,7 +4,7 @@ class CreateTagAliases < ActiveRecord::Migration
t.column :antecedent_name, :string, :null => false
t.column :consequent_name, :string, :null => false
t.column :creator_id, :integer, :null => false
t.column :request_ids, :string
t.column :forum_topic_id, :integer
t.timestamps
end

View File

@@ -5,7 +5,7 @@ class CreateTagImplications < ActiveRecord::Migration
t.column :consequent_name, :string, :null => false
t.column :descendant_names, :text, :null => false
t.column :creator_id, :integer, :null => false
t.column :request_ids, :string
t.column :forum_topic_id, :integer
t.timestamps
end

View File

@@ -1,16 +0,0 @@
class CreateJobs < ActiveRecord::Migration
def self.up
create_table :jobs do |t|
t.column :category, :string, :null => false
t.column :status, :string, :null => false
t.column :message, :text, :null => false
t.column :data_as_json, :text, :null => false
t.column :repeat_count, :integer, :null => false
t.timestamps
end
end
def self.down
drop_table :jobs
end
end

View File

@@ -0,0 +1,21 @@
class CreateDelayedJobs < ActiveRecord::Migration
def self.up
create_table :delayed_jobs, :force => true do |table|
table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue
table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually.
table.text :handler # YAML-encoded string of the object that will do work
table.text :last_error # reason for last failure (See Note below)
table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future.
table.datetime :locked_at # Set when a client is working on this object
table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead)
table.string :locked_by # Who is working on this object (if locked)
table.timestamps
end
add_index :delayed_jobs, :run_at
end
def self.down
drop_table :delayed_jobs
end
end

View File

@@ -1,8 +0,0 @@
require 'test_helper'
class JobsControllerTest < ActionController::TestCase
# Replace this with your real tests.
test "the truth" do
assert true
end
end

View File

@@ -1,6 +1,11 @@
require_relative "../test_helper"
class ArtistTest < ActiveSupport::TestCase
setup do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
context "The current user" do
should "be set only within the scope of the block" do
user = Factory.create(:user)
@@ -16,4 +21,29 @@ class ArtistTest < ActiveSupport::TestCase
assert_equal("1.2.3.4", CurrentUser.ip_addr)
end
end
context "A scoped current user" do
should "reset the current user after the block has exited" do
user1 = Factory.create(:user)
user2 = Factory.create(:user)
CurrentUser.user = user1
CurrentUser.scoped(user2, nil) do
assert_equal(user2.id, CurrentUser.user.id)
end
assert_equal(user1.id, CurrentUser.user.id)
end
should "reset the current user even if an exception is thrown" do
user1 = Factory.create(:user)
user2 = Factory.create(:user)
CurrentUser.user = user1
assert_raises(RuntimeError) do
CurrentUser.scoped(user2, nil) do
assert_equal(user2.id, CurrentUser.user.id)
raise "ERROR"
end
end
assert_equal(user1.id, CurrentUser.user.id)
end
end
end

View File

@@ -1,4 +0,0 @@
require 'test_helper'
class JobsHelperTest < ActionView::TestCase
end

View File

@@ -1,8 +1,23 @@
require_relative '../test_helper'
class JanitorTrialTest < ActiveSupport::TestCase
# Replace this with your real tests.
test "the truth" do
assert true
setup do
user = Factory.create(:user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
should "create a dmail when testing a new janitor" do
admin = Factory.create(:admin_user)
user = Factory.create(:user)
assert_difference("Dmail.count", 2) do
JanitorTrial.create(:user_id => user.id)
end
end
end

View File

@@ -1,8 +0,0 @@
require_relative '../test_helper'
class JobTest < ActiveSupport::TestCase
# Replace this with your real tests.
test "the truth" do
assert true
end
end