jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

changes

Browse Source
This commit is contained in:
albert
2010-08-27 16:59:59 -04:00
parent ad39553aac
commit 6bc469b05d
34 changed files with 299 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
app/models/janitor_trial.rb
View File

@@ -1,3 +1,16 @@
class JanitorTrial < ActiveRecord::Base
belongs_to :user
after_create :send_dmail
def send_dmail
body = "You have been selected as a test janitor. You can now approve pending posts and have access to the moderation interface.\n\nOver the next several weeks your approvals will be monitored. If the majority of them are quality uploads, then you will be promoted to full janitor status which grants you the ability to delete and undelete posts, ban users, and revert tag changes from vandals. If you fail the trial period, you will be demoted back to your original level and you'll receive a negative user record indicating you previously attempted and failed a test janitor trial.\n\nThere is a minimum quota of 5 approvals a week to indicate that you are being active. Remember, the goal isn't to approve as much as possible. It's to filter out borderline-quality art.\n\nIf you have any questions please respond to this message."
dmail = Dmail.new(
:title => "Test Janitor Trial Period",
:body => body
)
dmail.from_id = User.admins.first.id
dmail.to_id = user_id
Dmail.create_new(dmail)
end
end

168
app/models/job.rb
View File

@@ -1,168 +0,0 @@
class Job < ActiveRecord::Base
CATEGORIES = %w(mass_tag_edit approve_tag_alias approve_tag_implication calculate_tag_subscriptions calculate_related_tags s3_backup upload_processing)
STATUSES = %w(pending processing finished error)
validates_inclusion_of :category, :in => CATEGORIES
validates_inclusion_of :status, :in => STATUSES
def data
JSON.parse(data_as_json)
end
def data=(text)
self.data_as_json = text.to_json
end
def execute!
if repeat_count > 0
count = repeat_count - 1
else
count = repeat_count
end
begin
execute_sql("SET statement_timeout = 0")
update_attribute(:status, "processing")
__send__("execute_#{task_type}")
if count == 0
update_attribute(:status, "finished")
else
update_attributes(:status => "pending", :repeat_count => count)
end
rescue SystemExit => x
update_attribute(:status, "pending")
rescue Exception => x
update_attributes(:status => "error", :status_message => "#{x.class}: #{x}")
end
end
def execute_upload_processing
Upload.where("status = ?", "pending").each do |upload|
upload.process!
end
end
def execute_mass_tag_edit
start_tags = data["start_tags"]
result_tags = data["result_tags"]
updater_id = data["updater_id"]
updater_ip_addr = data["updater_ip_addr"]
Tag.mass_edit(start_tags, result_tags, updater_id, updater_ip_addr)
end
def execute_approve_tag_alias
ta = TagAlias.find(data["id"])
updater_id = data["updater_id"]
updater_ip_addr = data["updater_ip_addr"]
ta.approve(updater_id, updater_ip_addr)
end
def execute_approve_tag_implication
ti = TagImplication.find(data["id"])
updater_id = data["updater_id"]
updater_ip_addr = data["updater_ip_addr"]
ti.approve(updater_id, updater_ip_addr)
end
def execute_calculate_tag_subscriptions
last_run = Time.parse(data["last_run"])
if last_run.nil? || last_run < 20.minutes.ago
TagSubscription.process_all
update_attributes(:data => {:last_run => Time.now.strftime("%Y-%m-%d %H:%M")})
end
end
def execute_calculate_related_tags
tag_id = data["id"].to_i
tag = Tag.find_by_id(tag_id)
if tag
tag.commit_related(Tag.calculate_related(tag.name))
end
end
def execute_s3_backup
last_id = data["last_id"].to_i
begin
Post.where("id > ?", last_id).each do |post|
AWS::S3::Base.establish_connection!(
:access_key_id => Danbooru.config.amazon_s3_access_key_id,
:secret_access_key => Danbooru.config.amazon_s3_secret_access_key
)
if File.exists?(post.file_path)
AWS::S3::S3Object.store(
post.file_name,
open(post.file_path, "rb"),
Danbooru.config.amazon_s3_bucket_name,
"Content-MD5" => Base64.encode64(post.md5)
)
end
if post.image? && File.exists?(post.preview_path)
AWS::S3::S3Object.store(
"preview/#{post.md5}.jpg",
open(post.preview_path, "rb"),
Danbooru.config.amazon_s3_bucket_name
)
end
update_attributes(:data => {:last_id => post.id})
end
rescue Exception => x
# probably some network error, retry next time
end
end
def pretty_data
begin
case task_type
when "mass_tag_edit"
start = data["start_tags"]
result = data["result_tags"]
user = User.id_to_name(data["updater_id"])
"start:#{start} result:#{result} user:#{user}"
when "approve_tag_alias"
ta = TagAlias.find(data["id"])
"start:#{ta.name} result:#{ta.alias_name}"
when "approve_tag_implication"
ti = TagImplication.find(data["id"])
"start:#{ti.predicate.name} result:#{ti.consequent.name}"
when "calculate_tag_subscriptions"
last_run = data["last_run"]
"last run:#{last_run}"
when "calculate_related_tags"
tag = Tag.find_by_id(data["id"])
if tag
"tag:#{tag.name}"
else
"tag:UNKNOWN"
end
when "bandwidth_throttle"
""
when "s3_backup"
"last_id:" + data["last_id"].to_s
end
rescue Exception
"ERROR"
end
end
def self.pending_count(task_type)
where("task_type = ? and status = 'pending'", task_type).count
end
def self.execute_once
where("status = ?", "pending").each do |task|
task.execute!
sleep 1
end
end
end

27
app/models/jobs/backup_to_s3.rb Normal file
View File

@@ -0,0 +1,27 @@
module Jobs
class BackupToS3 < Struct.new(:last_id)
def perform
Post.find(:all, :conditions => ["id > ?", last_id], :limit => 200, :order => "id").each do |post|
AWS::S3::Base.establish_connection!(:access_key_id => CONFIG["amazon_s3_access_key_id"], :secret_access_key => CONFIG["amazon_s3_secret_access_key"])
if File.exists?(post.file_path)
base64_md5 = Base64.encode64(Digest::MD5.digest(File.read(post.file_path)))
AWS::S3::S3Object.store(post.file_name, open(post.file_path, "rb"), CONFIG["amazon_s3_bucket_name"], "Content-MD5" => base64_md5)
end
if post.image? && File.exists?(post.preview_path)
AWS::S3::S3Object.store("preview/#{post.md5}.jpg", open(post.preview_path, "rb"), CONFIG["amazon_s3_bucket_name"])
end
if File.exists?(post.sample_path)
AWS::S3::S3Object.store("sample/" + CONFIG["sample_filename_prefix"] + "#{post.md5}.jpg", open(post.sample_path, "rb"), CONFIG["amazon_s3_bucket_name"])
end
self.last_id = post.id
end
Delayed::Job.enqueue(BackupToS3.new(last_id))
rescue Exception => x
# probably some network error, retry next time
end
end
end

7
app/models/jobs/calculate_post_count.rb Normal file
View File

@@ -0,0 +1,7 @@
module Jobs
class CalculatePostCount < Struct.new(:tag_name)
def perform
Tag.recalculate_post_count(tag_name)
end
end
end

12
app/models/jobs/calculate_related_tags.rb Normal file
View File

@@ -0,0 +1,12 @@
module Jobs
class CalculateRelatedTags < Struct.new(:tag_id)
def perform
tag = Tag.find_by_id(tag_id)
if tag
tag.update_related
tag.save
end
end
end
end

13
app/models/jobs/calculate_uploaded_tags.rb Normal file
View File

@@ -0,0 +1,13 @@
module Jobs
class CalculateUploadedTags < Struct.new(:user_id)
def perform
tags = []
user = User.find(user_id)
CONFIG["tag_types"].values.uniq.each do |tag_type|
tags += user.calculate_uploaded_tags(tag_type)
end
user.update_attribute(:uploaded_tags, tags.join("\n"))
end
end
end

12
app/models/jobs/create_tag_alias.rb Normal file
View File

@@ -0,0 +1,12 @@
module Jobs
class CreateTagAlias < Struct.new(:antecedent_name, :consequent_name, :creator_id, :creator_ip_addr)
def execute
TagAlias.create(
:antecedent_name => antecedent_name,
:consequent_name => consequent_name,
:creator_id => creator_id,
:creator_ip_addr => creator_ip_addr
)
end
end
end

12
app/models/jobs/create_tag_implication.rb Normal file
View File

@@ -0,0 +1,12 @@
module Jobs
class CreateTagImplication < Struct.new(:antecedent_name, :consequent_name, :creator_id, :creator_ip_addr)
def perform
TagImplication.create(
:antecedent_name => antecedent_name,
:consequent_name => consequent_name,
:creator_id => creator_id,
:creator_ip_addr => creator_ip_addr
)
end
end
end

13
app/models/jobs/fix_pixiv_uploads.rb Normal file
View File

@@ -0,0 +1,13 @@
module Jobs
class FixPixivUploads < Struct.new(:last_post_id)
def perform
post_id = nil
Post.find_each(:conditions => ["GREATEST(width, height) IN (150, 600) AND source LIKE ? AND id > ?", "%pixiv%", last_post_id]) do |post|
post_id = post.id
end
update_attributes(:data => {:last_post_id => post_id})
end
end
end

7
app/models/jobs/mass_tag_edit.rb Normal file
View File

@@ -0,0 +1,7 @@
module Jobs
class MassTagEdit < Struct.new(:start_tags, :result_tags, :updater_id, :updater_ip_addr)
def perform
Tag.mass_edit(start_tags, result_tags, updater_id, updater_ip_addr)
end
end
end

10
app/models/jobs/process_tag_subscriptions.rb Normal file
View File

@@ -0,0 +1,10 @@
module Jobs
class ProcessTagSubscriptions < Struct.new(:last_run)
def perform
if last_run.nil? || last_run < 20.minutes.ago
TagSubscription.process_all
Delayed::Job.enqueue(ProcessTagSubscriptions.new(Time.now))
end
end
end
end

9
app/models/jobs/process_uploads.rb Normal file
View File

@@ -0,0 +1,9 @@
module Jobs
class ProcessUploads
def perform
Upload.find_each(:conditions => ["status = ?", "pending"]) do |upload|
upload.process!
end
end
end
end

9
app/models/tag.rb
View File

@@ -108,13 +108,14 @@ class Tag < ActiveRecord::Base
module UpdateMethods
def mass_edit(start_tags, result_tags, updater_id, updater_ip_addr)
raise NotImplementedError
updater = User.find(updater_id)
Post.find_by_tags(start_tags).each do |p|
start = TagAlias.to_aliased(scan_tags(start_tags))
result = TagAlias.to_aliased(scan_tags(result_tags))
tags = (p.cached_tags.scan(/\S+/) - start + result).join(" ")
p.update_attributes(:updater_user_id => updater_id, :updater_ip_addr => updater_ip_addr, :tags => tags)
tags = (p.tag_array - start + result).join(" ")
CurrentUser.scoped(updater, updater_ip_addr) do
p.update_attributes(:tag_string => tags)
end
end
end
end

16
app/models/tag_alias.rb
View File

@@ -1,11 +1,10 @@
class TagAlias < ActiveRecord::Base
attr_accessor :updater_id, :updater_ip_addr
attr_accessor :creator_ip_addr
after_save :update_posts
after_destroy :clear_cache
validates_presence_of :updater_id, :updater_ip_addr
validates_presence_of :creator_id, :creator_ip_addr
validates_uniqueness_of :antecedent_name
validate :absence_of_transitive_relation
belongs_to :updater, :class_name => "User"
belongs_to :creator, :class_name => "User"
def self.to_aliased(names)
@@ -41,11 +40,12 @@ class TagAlias < ActiveRecord::Base
Post.find_by_tags(antecedent_name).find_each do |post|
escaped_antecedent_name = Regexp.escape(antecedent_name)
fixed_tags = post.tag_string.sub(/(?:\A| )#{escaped_antecedent_name}(?:\Z| )/, " #{consequent_name} ").strip
post.update_attributes(
:tag_string => fixed_tags,
:updater_id => updater_id,
:updater_ip_addr => updater_ip_addr
)
CurrentUser.scoped(creator, creator_ip_addr) do
post.update_attributes(
:tag_string => fixed_tags
)
end
end
end
end

41
app/models/tag_implication.rb
View File

@@ -1,13 +1,12 @@
class TagImplication < ActiveRecord::Base
attr_accessor :updater_id, :updater_ip_addr
attr_accessor :creator_ip_addr
before_save :clear_cache
before_save :update_descendant_names
after_save :update_descendant_names_for_parent
after_destroy :clear_cache
after_save :update_posts
belongs_to :creator, :class_name => "User"
belongs_to :updater, :class_name => "User"
validates_presence_of :updater_id, :updater_ip_addr, :creator_id
validates_presence_of :creator_id, :creator_ip_addr
validates_uniqueness_of :antecedent_name, :scope => :consequent_name
validate :absence_of_circular_relation
@@ -30,15 +29,16 @@ class TagImplication < ActiveRecord::Base
end
def descendants
all = []
children = [consequent_name]
@descendants ||= begin
[].tap do |all|
children = [consequent_name]
until children.empty?
all += children
children = self.class.where(["antecedent_name IN (?)", children]).all.map(&:consequent_name)
until children.empty?
all += children
children = self.class.where(["antecedent_name IN (?)", children]).all.map(&:consequent_name)
end
end
end
all
end
def descendant_names_array
@@ -51,16 +51,14 @@ class TagImplication < ActiveRecord::Base
self.descendant_names = descendants.join(" ")
end
def update_descendant_names!(updater_id, updater_ip_addr)
def update_descendant_names!
update_descendant_names
self.updater_id = updater_id
self.updater_ip_addr = updater_ip_addr
save!
end
def update_descendant_names_for_parent
if parent
parent.update_descendant_names!(updater_id, updater_ip_addr)
parent.update_descendant_names!
end
end
@@ -72,17 +70,22 @@ class TagImplication < ActiveRecord::Base
Post.find_by_tags(antecedent_name).find_each do |post|
escaped_antecedent_name = Regexp.escape(antecedent_name)
fixed_tags = post.tag_string.sub(/(?:\A| )#{escaped_antecedent_name}(?:\Z| )/, " #{antecedent_name} #{descendant_names} ").strip
post.update_attributes(
:tag_string => fixed_tags,
:updater_id => updater_id,
:updater_ip_addr => updater_ip_addr
)
CurrentUser.scoped(creator, creator_ip_addr) do
post.update_attributes(
:tag_string => fixed_tags
)
end
end
end
def reload(options = {})
super
clear_parent_cache
clear_descendants_cache
end
def clear_descendants_cache
@descendants = nil
end
def clear_parent_cache

49
app/models/upload.rb
View File

@@ -49,33 +49,28 @@ class Upload < ActiveRecord::Base
module ConversionMethods
def process!
CurrentUser.user = uploader
CurrentUser.ip_addr = uploader_ip_addr
update_attribute(:status, "processing")
if is_downloadable?
download_from_source(temp_file_path)
CurrentUser.scoped(uploader, uploader_ip_addr) do
update_attribute(:status, "processing")
if is_downloadable?
download_from_source(temp_file_path)
end
validate_file_exists
self.file_ext = content_type_to_file_ext(content_type)
validate_file_content_type
calculate_hash(file_path)
validate_md5_uniqueness
validate_md5_confirmation
calculate_file_size(file_path)
calculate_dimensions(file_path) if has_dimensions?
generate_resizes(file_path)
move_file
post = convert_to_post
if post.save
update_attributes(:status => "completed", :post_id => post.id)
else
update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
end
end
validate_file_exists
self.file_ext = content_type_to_file_ext(content_type)
validate_file_content_type
calculate_hash(file_path)
validate_md5_uniqueness
validate_md5_confirmation
calculate_file_size(file_path)
calculate_dimensions(file_path) if has_dimensions?
generate_resizes(file_path)
move_file
post = convert_to_post
if post.save
update_attributes(:status => "completed", :post_id => post.id)
else
update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
end
rescue RuntimeError => x
ensure
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
def convert_to_post
@@ -85,8 +80,6 @@ class Upload < ActiveRecord::Base
p.file_ext = file_ext
p.image_width = image_width
p.image_height = image_height
p.uploader_id = uploader_id
p.uploader_ip_addr = uploader_ip_addr
p.rating = rating
p.source = source
p.file_size = file_size

3
app/models/user.rb
View File

@@ -22,7 +22,8 @@ class User < ActiveRecord::Base
has_many :feedback, :class_name => "UserFeedback", :dependent => :destroy
has_one :ban
belongs_to :inviter, :class_name => "User"
scope :named, lambda {|name| where(["lower(name) = ?", name])}
scope :named, lambda {|name| where(["lower(name) = ?", name])}
scope :admins, where("is_admin = TRUE")
module BanMethods
def validate_ip_addr_is_not_banned