Merge branch 'master' into fix-mention-dmails

app/models/amazon_backup.rb

@@ -12,6 +12,112 @@ class AmazonBackup < ActiveRecord::Base
     first.update_column(:last_id, new_id)
   end
 
+  def self.restore_from_glacier(min_id, max_id)
+    credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
+    Aws.config.update({
+      region: "us-east-1",
+      credentials: credentials
+    })
+    client = Aws::S3::Client.new
+    bucket = Danbooru.config.aws_s3_bucket_name
+
+    f = lambda do |key|
+      begin
+        client.restore_object(
+          bucket: bucket,
+          key: key,
+          restore_request: {
+            days: 7,
+            glacier_job_parameters: {
+              tier: "Bulk"
+            }
+          }
+        )
+      rescue Aws::S3::Errors::InternalError
+        puts "  internal error...retrying"
+        sleep 30
+        retry
+      rescue Aws::S3::Errors::InvalidObjectState
+        puts "  already restored #{key}"
+      rescue Aws::S3::Errors::NoSuchKey
+        puts "  missing #{key}"
+        file_path = "/var/www/danbooru2/shared/data/#{key}"
+
+        if File.exists?(file_path)
+          base64_md5 = Digest::MD5.base64digest(File.read(file_path))
+          body = open(file_path, "rb")
+          client.put_object(bucket: bucket, key: key, body: body, content_md5: base64_md5, acl: "public-read")
+          puts "  uploaded"
+        end
+      rescue Aws::S3::Errors::RestoreAlreadyInProgress
+        puts "  already restoring #{key}"
+      end
+    end
+
+    Post.where("id >= ? and id <= ?", min_id, max_id).find_each do |post|
+      if post.has_large?
+        puts "large:#{post.id}"
+        key = "sample/" + File.basename(post.large_file_path)
+        f.call(key)
+      end
+
+      if post.has_preview?
+        puts "preview:#{post.id}"
+        key = "preview/" + File.basename(post.preview_file_path)
+        f.call(key)
+      end
+
+      puts "#{post.id}"
+      key = File.basename(post.file_path)
+      f.call(key)
+    end
+  end
+
+  def self.copy_to_standard(min_id, max_id)
+    credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
+    Aws.config.update({
+      region: "us-east-1",
+      credentials: credentials
+    })
+    client = Aws::S3::Client.new
+    bucket = Danbooru.config.aws_s3_bucket_name
+
+    f = lambda do |key|
+      begin
+        client.copy_object(bucket: bucket, key: key, acl: "public-read", storage_class: "STANDARD", copy_source: "/#{bucket}/#{key}", metadata_directive: "COPY")
+        puts "  copied #{key}"
+      rescue Aws::S3::Errors::InternalError
+        puts "  internal error...retrying"
+        sleep 30
+        retry
+      rescue Aws::S3::Errors::InvalidObjectState
+        puts "  invalid state #{key}"
+      rescue Aws::S3::Errors::NoSuchKey
+        puts "  missing #{key}"
+      end
+    end
+
+    Post.where("id >= ? and id <= ?", min_id, max_id).find_each do |post|
+      next unless post.has_preview?
+
+      if post.has_preview?
+        puts "preview:#{post.id}"
+        key = "preview/" + File.basename(post.preview_file_path)
+        f.call(key)
+      end
+
+      if post.has_large?
+        puts "large:#{post.id}"
+        key = "sample/" + File.basename(post.large_file_path)
+        f.call(key)
+      end
+
+      puts "#{post.id}"
+      key = File.basename(post.file_path)
+      f.call(key)
+    end
+  end
+
   def self.execute
     return false unless Danbooru.config.aws_s3_enabled?
     

app/models/comment.rb

@@ -14,8 +14,8 @@ class Comment < ActiveRecord::Base
   after_update(:if => lambda {|rec| CurrentUser.id != rec.creator_id}) do |rec|
     ModAction.log("comment ##{rec.id} updated by #{CurrentUser.name}")
   end
-  after_destroy :update_last_commented_at_on_destroy
-  after_destroy(:if => lambda {|rec| CurrentUser.id != rec.creator_id}) do |rec|
+  after_update :update_last_commented_at_on_destroy, :if => lambda {|rec| rec.is_deleted? && rec.is_deleted_changed?}
+  after_update(:if => lambda {|rec| rec.is_deleted? && rec.is_deleted_changed? && CurrentUser.id != rec.creator_id}) do |rec|
     ModAction.log("comment ##{rec.id} deleted by #{CurrentUser.name}")
   end
   attr_accessible :body, :post_id, :do_not_bump_post, :is_deleted, :as => [:member, :gold, :platinum, :builder, :janitor, :moderator, :admin]

app/models/forum_topic.rb

@@ -154,6 +154,10 @@ class ForumTopic < ActiveRecord::Base
     self.updater_id = CurrentUser.id
   end
 
+  def page_for(post_id)
+    (posts.where("id < ?", post_id).count / Danbooru.config.posts_per_page.to_f).ceil
+  end
+
   def last_page
     (response_count / Danbooru.config.posts_per_page.to_f).ceil
   end

app/models/janitor_trial.rb

@@ -71,7 +71,7 @@ class JanitorTrial < ActiveRecord::Base
   end
 
   def send_dmail
-    body = "You have been selected as a test janitor. You can now approve pending posts and have access to the moderation interface. You should reacquaint yourself with the [[howto:upload]] guide to make sure you understand the site rules.\n\nOver the next several weeks your approvals will be monitored. If the majority of them are not quality uploads you will fail the trial period and lose your approval privileges. You will also receive a negative user record indicating you previously attempted and failed a test janitor trial.\n\nThere is a minimum quota of 1 approval a month to indicate that you are being active. Remember, the goal isn't to approve as much as possible. It's to filter out borderline-quality art.\n\nIf you have any questions please respond to this message."
+    body = "You have been selected as a test janitor. You can now approve pending posts and have access to the moderation interface. You should reacquaint yourself with the [[howto:upload]] guide to make sure you understand the site rules.\n\nOver the next several weeks your approvals will be monitored. If the majority of them are not quality uploads you will fail the trial period and lose your approval privileges. You will also receive a negative user record indicating you previously attempted and failed a test janitor trial.\n\nThere is a minimum quota of 1 approval a month to indicate that you are being active. Remember, the goal isn't to approve as much as possible. It's to filter out borderline-quality art."
 
     Dmail.create_automated(:title => "Test Janitor Trial Period", :body => body, :to_id => user_id)
   end

app/models/post.rb

@@ -42,7 +42,6 @@ class Post < ActiveRecord::Base
   has_one :pixiv_ugoira_frame_data, :class_name => "PixivUgoiraFrameData", :dependent => :destroy
   has_many :flags, :class_name => "PostFlag", :dependent => :destroy
   has_many :appeals, :class_name => "PostAppeal", :dependent => :destroy
-  has_many :versions, lambda {order("post_versions.updated_at ASC, post_versions.id ASC")}, :class_name => "PostVersion", :dependent => :destroy
   has_many :votes, :class_name => "PostVote", :dependent => :destroy
   has_many :notes, :dependent => :destroy
   has_many :comments, lambda {includes(:creator, :updater).order("comments.id")}, :dependent => :destroy
@@ -1414,12 +1413,16 @@ class Post < ActiveRecord::Base
   end
 
   module VersionMethods
+    def versions
+      if PostArchive.enabled?
+        PostArchive.where(post_id: id).order("updated_at ASC, id asc")
+      else
+        raise "Archive service not configured"
+      end
+    end
+
     def create_version(force = false)
       if new_record? || rating_changed? || source_changed? || parent_id_changed? || tag_string_changed? || force
-        if merge_version?
-          delete_previous_version
-        end
-
         create_new_version
       end
     end
@@ -1431,19 +1434,7 @@ class Post < ActiveRecord::Base
 
     def create_new_version
       User.where(id: CurrentUser.id).update_all("post_update_count = post_update_count + 1")
-      CurrentUser.reload
-
-      versions.create(
-        :rating => rating,
-        :source => source,
-        :tags => tag_string,
-        :parent_id => parent_id
-      )
-    end
-
-    def delete_previous_version
-      prev = versions.last
-      prev.destroy
+      PostArchive.queue(self) if PostArchive.enabled?
     end
 
     def revert_to(target)

app/models/post_archive.rb

@@ -1,4 +1,6 @@
 class PostArchive < ActiveRecord::Base
+  extend Memoist
+
   def self.enabled?
     Danbooru.config.aws_sqs_archives_url.present?
   end
@@ -6,49 +8,285 @@ class PostArchive < ActiveRecord::Base
   establish_connection (ENV["ARCHIVE_DATABASE_URL"] || "archive_#{Rails.env}".to_sym) if enabled?
   self.table_name = "post_versions"
 
-  def self.calculate_version(post_id, updated_at, version_id)
-    if updated_at.to_i == Time.zone.parse("2007-03-14T19:38:12Z").to_i
-      # Old post versions which didn't have updated_at set correctly
-      1 + PostVersion.where("post_id = ? and updated_at = ? and id < ?", post_id, updated_at, version_id).count
-    else
-      1 + PostVersion.where("post_id = ? and updated_at < ?", post_id, updated_at).count
-    end
-  end
-
-  def self.export(version_id = 9096768)
-    PostVersion.where("id > ?", version_id).find_each do |version|
-      previous = version.previous
-      tags = version.tags.scan(/\S+/)
-
-      if previous
-        prev_tags = previous.tags.scan(/\S+/)
-        added_tags = tags - previous.tags.scan(/\S+/)
-        removed_tags = previous.tags.scan(/\S+/) - tags
+  module SearchMethods
+    def for_user(user_id)
+      if user_id
+        where("updater_id = ?", user_id)
       else
-        added_tags = tags
-        removed_tags = []
+        where("false")
+      end
+    end
+
+    def for_user_name(name)
+      user_id = User.name_to_id(name)
+      for_user(user_id)
+    end
+
+    def search(params)
+      q = where("true")
+      params = {} if params.blank?
+
+      if params[:updater_name].present?
+        q = q.for_user_name(params[:updater_name])
       end
 
-      rating_changed = previous.nil? || version.rating != previous.try(:rating)
-      parent_changed = previous.nil? || version.parent_id != previous.try(:parent_id)
-      source_changed = previous.nil? || version.source != previous.try(:source)
-      create(
-        post_id: version.post_id,
-        tags: version.tags,
-        added_tags: added_tags,
-        removed_tags: removed_tags,
-        updater_id: version.updater_id,
-        updater_ip_addr: version.updater_ip_addr.to_s,
-        updated_at: version.updated_at,
-        version: calculate_version(version.post_id, version.updated_at, version.id),
-        rating: version.rating,
-        rating_changed: rating_changed,
-        parent_id: version.parent_id,
-        parent_changed: parent_changed,
-        source: version.source,
-        source_changed: source_changed
-      )
-      puts "inserted #{version.id}"
+      if params[:updater_id].present?
+        q = q.where("updater_id = ?", params[:updater_id].to_i)
+      end
+
+      if params[:post_id].present?
+        q = q.where("post_id = ?", params[:post_id].to_i)
+      end
+
+      if params[:start_id].present?
+        q = q.where("id <= ?", params[:start_id].to_i)
+      end
+
+      q
     end
   end
+
+  module ArchiveServiceMethods
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def sqs_service
+        SqsService.new(Danbooru.config.aws_sqs_archives_url)
+      end
+
+      def queue(post)
+        # queue updates to sqs so that if archives goes down for whatever reason it won't
+        # block post updates
+        raise "Archive service is not configured" if !enabled?
+
+        json = {
+          "post_id" => post.id,
+          "rating" => post.rating,
+          "parent_id" => post.parent_id,
+          "source" => post.source,
+          "updater_id" => CurrentUser.id,
+          "updater_ip_addr" => CurrentUser.ip_addr.to_s,
+          "updated_at" => post.updated_at.try(:iso8601),
+          "created_at" => post.created_at.try(:iso8601),
+          "tags" => post.tag_string
+        }
+        msg = "add post version\n#{json.to_json}"
+        sqs_service.send_message(msg)
+      end
+
+      def export_to_archives(version_id = 4394763)
+        PostVersion.where("id > ?", version_id).find_each do |version|
+          previous = version.previous
+          tags = version.tags.scan(/\S+/)
+          version_number = if version.updated_at.to_i == Time.zone.parse("2007-03-14T19:38:12Z").to_i
+            # Old post versions which didn't have updated_at set correctly
+            1 + PostVersion.where("post_id = ? and updated_at = ? and id < ?", version.post_id, version.updated_at, version.id).count
+          else
+            1 + PostVersion.where("post_id = ? and updated_at < ?", version.post_id, version.updated_at).count
+          end
+
+          if previous
+            prev_tags = previous.tags.scan(/\S+/)
+            added_tags = tags - prev_tags
+            removed_tags = prev_tags - tags
+          else
+            added_tags = tags
+            removed_tags = []
+          end
+
+          rating_changed = previous.nil? || version.rating != previous.rating
+          parent_changed = previous.nil? || version.parent_id != previous.parent_id
+          source_changed = previous.nil? || version.source != previous.source
+          create(
+            post_id: version.post_id,
+            tags: version.tags,
+            added_tags: added_tags,
+            removed_tags: removed_tags,
+            updater_id: version.updater_id,
+            updater_ip_addr: version.updater_ip_addr.to_s,
+            updated_at: version.updated_at,
+            version: version_number,
+            rating: version.rating,
+            rating_changed: rating_changed,
+            parent_id: version.parent_id,
+            parent_changed: parent_changed,
+            source: version.source,
+            source_changed: source_changed
+          )
+          puts "inserted #{version.id}"
+        end
+      end
+    end
+  end
+
+  extend SearchMethods
+  include ArchiveServiceMethods
+
+  def tag_array
+    tags.scan(/\S+/)
+  end
+
+  def presenter
+    PostVersionPresenter.new(self)
+  end
+
+  def reload
+    flush_cache
+    super
+  end
+
+  def post
+    Post.where(id: post_id).first
+  end
+
+  def previous
+    PostArchive.where("post_id = ? and version < ?", post_id, version).order("version desc").first
+  end
+
+  def updater
+    User.find(updater_id)
+  end
+
+  def diff(version = nil)
+    if post.nil?
+      latest_tags = tag_array
+    else
+      latest_tags = post.tag_array
+      latest_tags << "rating:#{post.rating}" if post.rating.present?
+      latest_tags << "parent:#{post.parent_id}" if post.parent_id.present?
+      latest_tags << "source:#{post.source}" if post.source.present?
+    end
+
+    new_tags = tag_array
+    new_tags << "rating:#{rating}" if rating.present?
+    new_tags << "parent:#{parent_id}" if parent_id.present?
+    new_tags << "source:#{source}" if source.present?
+
+    old_tags = version.present? ? version.tag_array : []
+    if version.present?
+      old_tags << "rating:#{version.rating}" if version.rating.present?
+      old_tags << "parent:#{version.parent_id}" if version.parent_id.present?
+      old_tags << "source:#{version.source}" if version.source.present?
+    end
+
+    added_tags = new_tags - old_tags
+    removed_tags = old_tags - new_tags
+
+    return {
+      :added_tags => added_tags,
+      :removed_tags => removed_tags,
+      :obsolete_added_tags => added_tags - latest_tags,
+      :obsolete_removed_tags => removed_tags & latest_tags,
+      :unchanged_tags => new_tags & old_tags,
+    }
+  end
+  
+  def changes
+    delta = {
+      :added_tags => added_tags,
+      :removed_tags => removed_tags
+    }
+
+    latest_tags = post.tag_array
+    latest_tags << "rating:#{post.rating}" if post.rating.present?
+    latest_tags << "parent:#{post.parent_id}" if post.parent_id.present?
+    latest_tags << "source:#{post.source}" if post.source.present?
+
+    if parent_changed
+      delta[:added_tags] << "parent:#{parent_id}"
+
+      if previous
+        delta[:removed_tags] << "parent:#{previous.parent_id}"
+      end
+    end
+
+    if rating_changed
+      delta[:added_tags] << "rating:#{rating}"
+
+      if previous
+        delta[:removed_tags] << "rating:#{previous.rating}"
+      end
+    end
+
+    if source_changed
+      delta[:added_tags] << "source:#{source}"
+
+      if previous
+        delta[:removed_tags] << "source:#{previous.source}"
+      end
+    end
+
+    delta[:obsolete_added_tags] = delta[:added_tags] - latest_tags
+    delta[:obsolete_removed_tags] = delta[:removed_tags] & latest_tags
+
+    if previous
+      delta[:unchanged_tags] = tag_array & previous.tag_array
+    else
+      delta[:unchanged_tags] = []
+    end
+
+    delta
+  end
+
+  def added_tags_with_fields
+    changes[:added_tags].join(" ")
+  end
+
+  def removed_tags_with_fields
+    changes[:removed_tags].join(" ")
+  end
+
+  def obsolete_added_tags
+    changes[:obsolete_added_tags].join(" ")
+  end
+
+  def obsolete_removed_tags
+    changes[:obsolete_removed_tags].join(" ")
+  end
+
+  def unchanged_tags
+    changes[:unchanged_tags].join(" ")
+  end
+
+  def truncated_source
+    source.gsub(/^http:\/\//, "").sub(/\/.+/, "")
+  end
+
+  def undo
+    added = changes[:added_tags_with_fields] - changes[:obsolete_added_tags]
+    removed = changes[:removed_tags_with_fields] - changes[:obsolete_removed_tags]
+
+    added.each do |tag|
+      if tag =~ /^source:/
+        post.source = ""
+      elsif tag =~ /^parent:/
+        post.parent_id = nil
+      else
+        escaped_tag = Regexp.escape(tag)
+        post.tag_string = post.tag_string.sub(/(?:\A| )#{escaped_tag}(?:\Z| )/, " ").strip
+      end
+    end
+    removed.each do |tag|
+      if tag =~ /^source:(.+)$/
+        post.source = $1
+      else
+        post.tag_string = "#{post.tag_string} #{tag}".strip
+      end
+    end
+  end
+
+  def undo!
+    undo
+    post.save!
+  end
+
+  def updater
+    User.find_by_id(updater_id)
+  end
+
+  def method_attributes
+    super + [:obsolete_added_tags, :obsolete_removed_tags, :unchanged_tags, :updater_name]
+  end
+
+  memoize :previous, :post, :tag_array, :changes, :added_tags_with_fields, :removed_tags_with_fields, :obsolete_removed_tags, :obsolete_added_tags, :unchanged_tags
 end

app/models/post_version.rb

@@ -61,17 +61,6 @@ class PostVersion < ActiveRecord::Base
     super
   end
 
-  def sequence_for_post
-    versions = PostVersion.where(:post_id => post_id).order("updated_at desc, id desc")
-    diffs = []
-    versions.each_index do |i|
-      if i < versions.size - 1
-        diffs << versions[i].diff(versions[i + 1])
-      end
-    end
-    return diffs
-  end
-
   def diff(version)
     latest_tags = post.tag_array
     latest_tags << "rating:#{post.rating}" if post.rating.present?

app/models/user.rb

@@ -54,10 +54,8 @@ class User < ActiveRecord::Base
   attr_accessor :password, :old_password
   attr_accessible :dmail_filter_attributes, :enable_privacy_mode, :enable_post_navigation, :new_post_navigation_layout, :password, :old_password, :password_confirmation, :password_hash, :email, :last_logged_in_at, :last_forum_read_at, :has_mail, :receive_email_notifications, :comment_threshold, :always_resize_images, :favorite_tags, :blacklisted_tags, :name, :ip_addr, :time_zone, :default_image_size, :enable_sequential_post_navigation, :per_page, :hide_deleted_posts, :style_usernames, :enable_auto_complete, :custom_style, :show_deleted_children, :disable_categorized_saved_searches, :disable_tagged_filenames, :enable_recent_searches, :as => [:moderator, :janitor, :gold, :platinum, :member, :anonymous, :default, :builder, :admin]
   attr_accessible :level, :as => :admin
-  validates_length_of :name, :within => 2..100, :on => :create
-  validates_format_of :name, :with => /\A[^\s:]+\Z/, :on => :create, :message => "cannot have whitespace or colons"
-  validates_format_of :name, :with => /\A[^_].*[^_]\Z/, :on => :create, :message => "cannot begin or end with an underscore"
-  validates_uniqueness_of :name, :case_sensitive => false
+
+  validates :name, user_name: true, on: :create
   validates_uniqueness_of :email, :case_sensitive => false, :if => lambda {|rec| rec.email.present? && rec.email_changed? }
   validates_length_of :password, :minimum => 5, :if => lambda {|rec| rec.new_record? || rec.password.present?}
   validates_inclusion_of :default_image_size, :in => %w(large original)
@@ -153,6 +151,10 @@ class User < ActiveRecord::Base
       def id_to_pretty_name(user_id)
         id_to_name(user_id).gsub(/([^_])_+(?=[^_])/, "\\1 \\2")
       end
+
+      def normalize_name(name)
+        name.to_s.mb_chars.downcase.strip.tr(" ", "_").to_s
+      end
     end
 
     def pretty_name

app/models/user_name_change_request.rb

@@ -3,11 +3,8 @@ class UserNameChangeRequest < ActiveRecord::Base
   validates_inclusion_of :status, :in => %w(pending approved rejected)
   belongs_to :user
   belongs_to :approver, :class_name => "User"
-  validate :uniqueness_of_desired_name
   validate :not_limited, :on => :create
-  validates_length_of :desired_name, :within => 2..100, :on => :create
-  validates_format_of :desired_name, :with => /\A[^\s:]+\Z/, :on => :create, :message => "cannot have whitespace or colons"
-  before_validation :normalize_name
+  validates :desired_name, user_name: true
   attr_accessible :status, :user_id, :original_name, :desired_name, :change_reason, :rejection_reason, :approver_id
   
   def self.pending
@@ -40,8 +37,8 @@ class UserNameChangeRequest < ActiveRecord::Base
     status == "pending"
   end
   
-  def normalize_name
-    self.desired_name = desired_name.strip.gsub(/ /, "_")
+  def desired_name=(name)
+    super(User.normalize_name(name))
   end
   
   def feedback
@@ -71,15 +68,6 @@ class UserNameChangeRequest < ActiveRecord::Base
       return true
     end
   end
-  
-  def uniqueness_of_desired_name
-    if User.find_by_name(desired_name)
-      errors.add(:desired_name, "already exists")
-      return false
-    else
-      return true
-    end
-  end
 
   def hidden_attributes
     if CurrentUser.is_admin? || user == CurrentUser.user

app/models/wiki_page.rb

@@ -163,6 +163,7 @@ class WikiPage < ActiveRecord::Base
       :title => title,
       :body => body,
       :is_locked => is_locked,
+      :is_deleted => is_deleted,
       :other_names => other_names
     )
   end

app/models/wiki_page_version.rb

@@ -2,7 +2,7 @@ class WikiPageVersion < ActiveRecord::Base
   belongs_to :wiki_page
   belongs_to :updater, :class_name => "User"
   belongs_to :artist
-  attr_accessible :wiki_page_id, :title, :body, :is_locked, :updater_id, :updater_ip_addr, :version, :other_names
+  attr_accessible :wiki_page_id, :title, :body, :is_locked, :is_deleted, :updater_id, :updater_ip_addr, :version, :other_names
   delegate :visible?, :to => :wiki_page
 
   module SearchMethods