Merge branch 'master' into fix-recaptcha

Gemfile.lock

@@ -64,8 +64,8 @@ GEM
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    addressable (2.5.1)
+    addressable (2.5.2)
-      public_suffix (~> 2.0, >= 2.0.2)
+      public_suffix (>= 2.0.2, < 4.0)
     arel (6.0.4)
     awesome_print (1.7.0)
     aws-sdk (2.7.4)
@@ -108,7 +108,7 @@ GEM
     cityhash (0.8.1)
     coderay (1.1.1)
     colorize (0.7.7)
-    crack (0.4.2)
+    crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.2)
     daemons (1.2.3)
@@ -156,6 +156,7 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
+    hashdiff (0.3.7)
     highline (1.7.8)
     hike (1.2.3)
     http (2.2.2)
@@ -246,7 +247,7 @@ GEM
     pry-byebug (3.4.2)
       byebug (~> 9.0)
       pry (~> 0.10)
-    public_suffix (2.0.5)
+    public_suffix (3.0.1)
     rack (1.6.5)
     rack-test (0.6.3)
       rack (>= 1.0)
@@ -385,9 +386,10 @@ GEM
     unicorn-worker-killer (0.4.4)
       get_process_mem (~> 0)
       unicorn (>= 4, < 6)
-    webmock (1.21.0)
+    webmock (3.1.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
+      hashdiff
     webrobots (0.1.2)
     whenever (0.9.7)
       chronic (>= 0.6.3)

app/assets/javascripts/autocomplete.js

@@ -244,7 +244,7 @@
     $.ajax({
       url: "/tags/autocomplete.json",
       data: {
-        "search[name_matches]": term + "*"
+        "search[name_matches]": term
       },
       method: "get",
       success: function(data) {

app/controllers/posts_controller.rb

@@ -110,6 +110,10 @@ private
   def respond_with_post_after_update(post)
     respond_with(post) do |format|
       format.html do
+        if post.warnings.any?
+          flash[:notice] = post.warnings.full_messages.join(".\n \n")
+        end
+
         if post.errors.any?
           @error_message = post.errors.full_messages.join("; ")
           render :template => "static/error", :status => 500

app/controllers/uploads_controller.rb

@@ -53,7 +53,12 @@ class UploadsController < ApplicationController
 
   def create
     @upload = Upload.create(params[:upload].merge(:server => Socket.gethostname))
-    @upload.process! if @upload.errors.empty?
+
+    if @upload.errors.empty?
+      post = @upload.process!
+      flash[:notice] = post.warnings.full_messages.join(".\n \n") if post.present? && post.warnings.any?
+    end
+
     save_recent_tags
     respond_with(@upload)
   end

app/logical/bulk_revert.rb

@@ -10,7 +10,7 @@ class BulkRevert
     ModAction.log("Processed bulk revert for #{constraints.inspect} by #{creator.name}")
 
     CurrentUser.scoped(creator) do
-      ActiveRecord::Base.without_timeout do
+      ApplicationRecord.without_timeout do
         find_post_versions.order("updated_at, id").each do |version|
           version.undo!
         end

app/logical/moderator/dashboard/report.rb

@@ -9,37 +9,37 @@ module Moderator
       end
 
       def artists
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::Artist.all(min_date, max_level)
         end
       end
 
       def comments
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::Comment.all(min_date, max_level)
         end
       end
 
       def mod_actions
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::ModAction.all
         end
       end
 
       def notes
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::Note.all(min_date, max_level)
         end
       end
 
       def appeals
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::PostAppeal.all(min_date)
         end
       end
 
       def flags
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::PostFlag.all(min_date)
         end
       end
@@ -49,19 +49,19 @@ module Moderator
       end
 
       def posts
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::Upload.all(min_date, max_level)
         end
       end
 
       def user_feedbacks
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::UserFeedback.all
         end
       end
 
       def wiki_pages
-        ActiveRecord::Base.without_timeout do
+        ApplicationRecord.without_timeout do
           Queries::WikiPage.all(min_date, max_level)
         end
       end

app/logical/moderator/tag_batch_change.rb

@@ -18,11 +18,12 @@ module Moderator
             post.update_attributes(:tag_string => tags)
           end
 
-          tags = Tag.scan_tags(antecedent, :strip_metatags => true)
-          conds = tags.map {|x| "query like ?"}.join(" AND ")
-          conds = [conds, *tags.map {|x| "%#{x.to_escaped_for_sql_like}%"}]
           if SavedSearch.enabled?
-            SavedSearch.where(*conds).find_each do |ss|
+            tags = Tag.scan_tags(antecedent, :strip_metatags => true)
+
+            # https://www.postgresql.org/docs/current/static/functions-array.html
+            saved_searches = SavedSearch.where("string_to_array(query, ' ') @> ARRAY[?]", tags)
+            saved_searches.find_each do |ss|
               ss.query = (ss.query.split - tags + [consequent]).uniq.join(" ")
               ss.save
             end

app/logical/post_sets/post.rb

@@ -87,11 +87,15 @@ module PostSets
     end
 
     def banned_posts
-      posts.select(&:is_banned?)
+      posts.select { |p| p.banblocked? }
     end
 
     def censored_posts
-      hidden_posts - banned_posts
+      posts.select { |p| p.levelblocked? && !p.banblocked? }
+    end
+
+    def safe_posts
+      posts.select { |p| p.safeblocked? && !p.levelblocked? && !p.banblocked? }
     end
 
     def use_sequential_paginator?

app/models/application_record.rb

@@ -43,5 +43,50 @@ class ApplicationRecord < ActiveRecord::Base
     end
   end
 
+  concerning :ActiveRecordExtensions do
+    class_methods do
+      def without_timeout
+        connection.execute("SET STATEMENT_TIMEOUT = 0") unless Rails.env == "test"
+        yield
+      ensure
+        connection.execute("SET STATEMENT_TIMEOUT = #{CurrentUser.user.try(:statement_timeout) || 3_000}") unless Rails.env == "test"
+      end
+
+      def with_timeout(n, default_value = nil, new_relic_params = {})
+        connection.execute("SET STATEMENT_TIMEOUT = #{n}") unless Rails.env == "test"
+        yield
+      rescue ::ActiveRecord::StatementInvalid => x
+        if Rails.env.production?
+          NewRelic::Agent.notice_error(x, :custom_params => new_relic_params.merge(:user_id => CurrentUser.id, :user_ip_addr => CurrentUser.ip_addr))
+        end
+        return default_value
+      ensure
+        connection.execute("SET STATEMENT_TIMEOUT = #{CurrentUser.user.try(:statement_timeout) || 3_000}") unless Rails.env == "test"
+      end
+    end
+
+    %w(execute select_value select_values select_all).each do |method_name|
+      define_method("#{method_name}_sql") do |sql, *params|
+        self.class.connection.__send__(method_name, self.class.send(:sanitize_sql_array, [sql, *params]))
+      end
+
+      self.class.__send__(:define_method, "#{method_name}_sql") do |sql, *params|
+        connection.__send__(method_name, send(:sanitize_sql_array, [sql, *params]))
+      end
+    end
+  end
+
+  concerning :PostgresExtensions do
+    class_methods do
+      def columns(*params)
+        super.reject {|x| x.sql_type == "tsvector"}
+      end
+    end
+  end
+
+  def warnings
+    @warnings ||= ActiveModel::Errors.new(self)
+  end
+
   include ApiMethods
 end

app/models/comment.rb

@@ -182,8 +182,8 @@ class Comment < ApplicationRecord
   end
 
   def initialize_updater
-    self.updater_id ||= CurrentUser.user.id
+    self.updater_id = CurrentUser.user.id
-    self.updater_ip_addr ||= CurrentUser.ip_addr
+    self.updater_ip_addr = CurrentUser.ip_addr
   end
 
   def creator_name
@@ -260,5 +260,3 @@ class Comment < ApplicationRecord
     DText.quote(body, creator_name)
   end
 end
-
-Comment.connection.extend(PostgresExtensions)

app/models/dmail.rb

@@ -30,6 +30,12 @@ class Dmail < ApplicationRecord
     def creator_ip_addr_str
       creator_ip_addr.to_s
     end
+
+    def spam?(sender = CurrentUser.user)
+      return false if Danbooru.config.rakismet_key.blank?
+      return false if sender.is_gold?
+      super()
+    end
   end
 
   module AddressMethods
@@ -52,12 +58,7 @@ class Dmail < ApplicationRecord
     def initialize_attributes
       self.from_id ||= CurrentUser.id
       self.creator_ip_addr ||= CurrentUser.ip_addr
-      if CurrentUser.is_gold?
+      self.is_spam = spam?(CurrentUser.user)
-        self.is_spam = false
-      else
-        self.is_spam = spam?
-      end
-      true
     end
   end
 

app/models/favorite_group.rb

@@ -173,11 +173,13 @@ class FavoriteGroup < ApplicationRecord
   end
 
   def add!(post_id)
-    post_id = post_id.id if post_id.is_a?(Post)
+    with_lock do
-    return if contains?(post_id)
+      post_id = post_id.id if post_id.is_a?(Post)
+      return if contains?(post_id)
 
-    clear_post_id_array
+      clear_post_id_array
-    update_attributes(:post_ids => add_number_to_string(post_id, post_ids))
+      update_attributes(:post_ids => add_number_to_string(post_id, post_ids))
+    end
   end
 
   def self.purge_post(post_id)
@@ -188,11 +190,13 @@ class FavoriteGroup < ApplicationRecord
   end
 
   def remove!(post_id)
-    post_id = post_id.id if post_id.is_a?(Post)
+    with_lock do
-    return unless contains?(post_id)
+      post_id = post_id.id if post_id.is_a?(Post)
+      return unless contains?(post_id)
 
-    clear_post_id_array
+      clear_post_id_array
-    update_attributes(:post_ids => remove_number_from_string(post_id, post_ids))
+      update_attributes(:post_ids => remove_number_from_string(post_id, post_ids))
+    end
   end
 
   def add_number_to_string(number, string)

app/models/post.rb

@@ -18,6 +18,10 @@ class Post < ApplicationRecord
   validates_uniqueness_of :md5, :on => :create
   validates_inclusion_of :rating, in: %w(s q e), message: "rating must be s, q, or e"
   validate :tag_names_are_valid
+  validate :added_tags_are_valid
+  validate :removed_tags_are_valid
+  validate :has_artist_tag
+  validate :has_copyright_tag
   validate :post_is_not_its_own_parent
   validate :updater_can_change_rating
   before_save :update_tag_post_counts
@@ -196,6 +200,14 @@ class Post < ApplicationRecord
       "http://#{Danbooru.config.hostname}#{preview_file_url}"
     end
 
+    def open_graph_image_url
+      if is_image? && has_large?
+        "http://#{Danbooru.config.hostname}#{large_file_url}"
+      else
+        complete_preview_file_url
+      end
+    end
+
     def file_url_for(user)
       if CurrentUser.mobile_mode?
         large_file_url
@@ -592,6 +604,18 @@ class Post < ApplicationRecord
       @tag_array_was ||= Tag.scan_tags(tag_string_was)
     end
 
+    def tags
+      Tag.where(name: tag_array)
+    end
+
+    def tags_was
+      Tag.where(name: tag_array_was)
+    end
+
+    def added_tags
+      tags - tags_was
+    end
+
     def decrement_tag_post_counts
       Tag.where(:name => tag_array).update_all("post_count = post_count - 1") if tag_array.any?
     end
@@ -633,12 +657,18 @@ class Post < ApplicationRecord
     end
 
     def merge_old_changes
+      @removed_tags = []
+
       if old_tag_string
         # If someone else committed changes to this post before we did,
         # then try to merge the tag changes together.
         current_tags = tag_array_was()
         new_tags = tag_array()
         old_tags = Tag.scan_tags(old_tag_string)
+
+        kept_tags = current_tags & new_tags
+        @removed_tags = old_tags - kept_tags
+
         set_tag_string(((current_tags + new_tags) - old_tags + (current_tags & new_tags)).uniq.sort.join(" "))
       end
 
@@ -687,10 +717,10 @@ class Post < ApplicationRecord
     end
 
     def remove_negated_tags(tags)
-      negated_tags, tags = tags.partition {|x| x =~ /\A-/i}
+      @negated_tags, tags = tags.partition {|x| x =~ /\A-/i}
-      negated_tags = negated_tags.map {|x| x[1..-1]}
+      @negated_tags = @negated_tags.map {|x| x[1..-1]}
-      negated_tags = TagAlias.to_aliased(negated_tags)
+      @negated_tags = TagAlias.to_aliased(@negated_tags)
-      return tags - negated_tags
+      return tags - @negated_tags
     end
 
     def add_automatic_tags(tags)
@@ -1369,18 +1399,15 @@ class Post < ApplicationRecord
       Post.transaction do
         flag!(reason, is_deletion: true)
 
-        self.is_deleted = true
+        update({
-        self.is_pending = false
+          is_deleted: true,
-        self.is_flagged = false
+          is_pending: false,
-        self.is_banned = true if options[:ban] || has_tag?("banned_artist")
+          is_flagged: false,
-        update_columns(
+          is_banned: is_banned || options[:ban] || has_tag?("banned_artist")
-          :is_deleted => is_deleted,
+        }, without_protection: true)
-          :is_pending => is_pending,
+
-          :is_flagged => is_flagged,
+        # XXX This must happen *after* the `is_deleted` flag is set to true (issue #3419).
-          :is_banned => is_banned
-        )
         give_favorites_to_parent if options[:move_favorites]
-        update_parent_on_save
 
         unless options[:without_mod_action]
           ModAction.log("deleted post ##{id}, reason: #{reason}")
@@ -1708,6 +1735,53 @@ class Post < ApplicationRecord
         end
       end
     end
+
+    def added_tags_are_valid
+      new_tags = added_tags.select { |t| t.post_count <= 1 }
+      new_general_tags = new_tags.select { |t| t.category == Tag.categories.general }
+      new_artist_tags = new_tags.select { |t| t.category == Tag.categories.artist }
+
+      if new_general_tags.present?
+        n = new_general_tags.size
+        tag_wiki_links = new_general_tags.map { |tag| "[[#{tag.name}]]" }
+        self.warnings[:base] << "Created #{n} new #{n == 1 ? "tag" : "tags"}: #{tag_wiki_links.join(", ")}"
+      end
+
+      new_artist_tags.each do |tag|
+        if tag.artist.blank?
+          self.warnings[:base] << "Artist [[#{tag.name}]] requires an artist entry. \"Create new artist entry\":[/artists/new?name=#{CGI::escape(tag.name)}]"
+        end
+      end
+    end
+
+    def removed_tags_are_valid
+      attempted_removed_tags = @removed_tags + @negated_tags
+      unremoved_tags = tag_array & attempted_removed_tags
+
+      if unremoved_tags.present?
+        unremoved_tags_list = unremoved_tags.map { |t| "[[#{t}]]" }.to_sentence
+        self.warnings[:base] << "#{unremoved_tags_list} could not be removed. Check for implications and try again"
+      end
+    end
+
+    def has_artist_tag
+      return if !new_record?
+      return if source !~ %r!\Ahttps?://!
+      return if has_tag?("artist_request") || has_tag?("official_art")
+      return if tags.any? { |t| t.category == Tag.categories.artist }
+
+      site = Sources::Site.new(source)
+      self.warnings[:base] << "Artist tag is required. Create a new tag with [[artist:<artist_name>]]. Ask on the forum if you need naming help"
+    rescue Sources::Site::NoStrategyError => e
+      # unrecognized source; do nothing.
+    end
+
+    def has_copyright_tag
+      return if !new_record?
+      return if has_tag?("copyright_request") || tags.any? { |t| t.category == Tag.categories.copyright }
+
+      self.warnings[:base] << "Copyright tag is required. Consider adding [[copyright request]] or [[original]]"
+    end
   end
   
   include FileMethods
@@ -1738,11 +1812,22 @@ class Post < ApplicationRecord
   )
   has_bit_flags BOOLEAN_ATTRIBUTES
 
+  def safeblocked?
+    CurrentUser.safe_mode? && (rating != "s" || has_tag?("toddlercon|toddler|diaper|tentacle|rape|bestiality|beastiality|lolita|loli|nude|shota|pussy|penis"))
+  end
+
+  def levelblocked?
+    !Danbooru.config.can_user_see_post?(CurrentUser.user, self)
+  end
+
+  def banblocked?
+    is_banned? && !CurrentUser.is_gold?
+  end
+
   def visible?
-    return false if !Danbooru.config.can_user_see_post?(CurrentUser.user, self)
+    return false if safeblocked?
-    return false if CurrentUser.safe_mode? && rating != "s"
+    return false if levelblocked?
-    return false if CurrentUser.safe_mode? && has_tag?("toddlercon|toddler|diaper|tentacle|rape|bestiality|beastiality|lolita|loli|nude|shota|pussy|penis")
+    return false if banblocked?
-    return false if is_banned? && !CurrentUser.is_gold?
     return true
   end
 
@@ -1798,5 +1883,3 @@ class Post < ApplicationRecord
     ret
   end
 end
-
-Post.connection.extend(PostgresExtensions)

app/models/tag.rb

@@ -6,6 +6,7 @@ class Tag < ApplicationRecord
   attr_accessible :category, :as => [:moderator, :gold, :platinum, :member, :anonymous, :default, :builder, :admin]
   attr_accessible :is_locked, :as => [:moderator, :admin]
   has_one :wiki_page, :foreign_key => "title", :primary_key => "name"
+  has_one :artist, :foreign_key => "name", :primary_key => "name"
   has_one :antecedent_alias, lambda {active}, :class_name => "TagAlias", :foreign_key => "antecedent_name", :primary_key => "name"
   has_many :consequent_aliases, lambda {active}, :class_name => "TagAlias", :foreign_key => "consequent_name", :primary_key => "name"
   has_many :antecedent_implications, lambda {active}, :class_name => "TagImplication", :foreign_key => "antecedent_name", :primary_key => "name"
@@ -816,6 +817,18 @@ class Tag < ApplicationRecord
       where("tags.post_count > 0")
     end
 
+    # ref: https://www.postgresql.org/docs/current/static/pgtrgm.html#idm46428634524336
+    def order_similarity(name)
+      # trunc(3 * sim) reduces the similarity score from a range of 0.0 -> 1.0 to just 0, 1, or 2.
+      # This groups tags first by approximate similarity, then by largest tags within groups of similar tags.
+      order("trunc(3 * similarity(name, #{sanitize(name)})) DESC", "post_count DESC", "name DESC")
+    end
+
+    # ref: https://www.postgresql.org/docs/current/static/pgtrgm.html#idm46428634524336
+    def fuzzy_name_matches(name)
+      where("tags.name % ?", name)
+    end
+
     def name_matches(name)
       where("tags.name LIKE ? ESCAPE E'\\\\'", normalize_name(name).to_escaped_for_sql_like)
     end
@@ -828,6 +841,10 @@ class Tag < ApplicationRecord
       q = where("true")
       params = {} if params.blank?
 
+      if params[:fuzzy_name_matches].present?
+        q = q.fuzzy_name_matches(params[:fuzzy_name_matches])
+      end
+
       if params[:name_matches].present?
         q = q.name_matches(params[:name_matches])
       end
@@ -864,6 +881,8 @@ class Tag < ApplicationRecord
         q = q.reorder("id desc")
       when "count"
         q = q.reorder("post_count desc")
+      when "similarity"
+        q = q.order_similarity(params[:fuzzy_name_matches]) if params[:fuzzy_name_matches].present?
       else
         q = q.reorder("id desc")
       end
@@ -872,21 +891,29 @@ class Tag < ApplicationRecord
     end
 
     def names_matches_with_aliases(name)
-      query1 = Tag.select("tags.name, tags.post_count, tags.category, null AS antecedent_name")
+      name = normalize_name(name)
-        .search(:name_matches => name, :order => "count").limit(10)
+      wildcard_name = name + '*'
+
+      query1 = Tag.select("tags.name, tags.post_count, tags.category, null AS antecedent_name")
+        .search(:name_matches => wildcard_name, :order => "count").limit(10)
 
-      name = name.mb_chars.downcase.to_escaped_for_sql_like
       query2 = TagAlias.select("tags.name, tags.post_count, tags.category, tag_aliases.antecedent_name")
         .joins("INNER JOIN tags ON tags.name = tag_aliases.consequent_name")
-        .where("tag_aliases.antecedent_name LIKE ? ESCAPE E'\\\\'", name)
+        .where("tag_aliases.antecedent_name LIKE ? ESCAPE E'\\\\'", wildcard_name.to_escaped_for_sql_like)
         .active
-        .where("tags.name NOT LIKE ? ESCAPE E'\\\\'", name)
+        .where("tags.name NOT LIKE ? ESCAPE E'\\\\'", wildcard_name.to_escaped_for_sql_like)
         .where("tag_aliases.post_count > 0")
         .order("tag_aliases.post_count desc")
         .limit(20) # Get 20 records even though only 10 will be displayed in case some duplicates get filtered out.
 
       sql_query = "((#{query1.to_sql}) UNION ALL (#{query2.to_sql})) AS unioned_query"
-      Tag.select("DISTINCT ON (name, post_count) *").from(sql_query).order("post_count desc").limit(10)
+      tags = Tag.select("DISTINCT ON (name, post_count) *").from(sql_query).order("post_count desc").limit(10)
+
+      if tags.empty?
+        tags = Tag.fuzzy_name_matches(name).order_similarity(name).nonempty.limit(10)
+      end
+
+      tags
     end
   end
 

app/models/tag_implication.rb

@@ -21,7 +21,7 @@ class TagImplication < TagRelationship
       end
 
       def automatic_tags_for(names)
-        tags = names.grep(/\A(.+)_\(cosplay\)\Z/) { "char:#{$1}" }
+        tags = names.grep(/\A(.+)_\(cosplay\)\Z/) { "char:#{TagAlias.to_aliased([$1]).first}" }
         tags << "cosplay" if tags.present?
         tags.uniq
       end

app/models/upload.rb

@@ -148,6 +148,8 @@ class Upload < ApplicationRecord
       else
         update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
       end
+
+      post
     end
 
     def process!(force = false)
@@ -155,7 +157,7 @@ class Upload < ApplicationRecord
       return if !force && status =~ /processing|completed|error/
 
       process_upload
-      create_post_from_upload
+      post = create_post_from_upload
 
     rescue Timeout::Error, Net::HTTP::Persistent::Error => x
       if @tries > 3
@@ -164,9 +166,11 @@ class Upload < ApplicationRecord
         @tries += 1
         retry
       end
+      nil
 
     rescue Exception => x
       update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
+      nil
       
     ensure
       delete_temp_file

app/presenters/post_presenter.rb

@@ -155,10 +155,19 @@ class PostPresenter < Presenter
     categorized_tag_groups.flatten.slice(0, 25).join(", ").tr("_", " ")
   end
 
+  def safe_mode_message(template)
+    html = ["This image is unavailable on safe mode (#{Danbooru.config.app_name}). Go to "]
+    html << template.link_to("Danbooru", "http://danbooru.donmai.us")
+    html << " or disable safe mode to view ("
+    html << template.link_to("learn more", template.wiki_pages_path(title: "help:user_settings"))
+    html << ")."
+    html.join.html_safe
+  end
+
   def image_html(template)
-    return template.content_tag("p", "The artist requested removal of this image") if @post.is_banned? && !CurrentUser.user.is_gold?
+    return template.content_tag("p", "The artist requested removal of this image") if @post.banblocked?
-    return template.content_tag("p", template.link_to("You need a gold account to see this image.", template.new_user_upgrade_path)) if !Danbooru.config.can_user_see_post?(CurrentUser.user, @post)
+    return template.content_tag("p", template.link_to("You need a gold account to see this image.", template.new_user_upgrade_path)) if @post.levelblocked?
-    return template.content_tag("p", "This image is unavailable") if !@post.visible?
+    return template.content_tag("p", safe_mode_message(template)) if @post.safeblocked?
 
     if @post.is_flash?
       template.render("posts/partials/show/flash", :post => @post)

app/views/layouts/default.html.erb

@@ -112,7 +112,7 @@
     <% end %>
 
     <div class="ui-corner-all ui-state-highlight" id="notice" style="<%= "display: none;" unless flash[:notice] %>">
-      <span><%= flash[:notice] %></span>
+      <span><%= format_text(flash[:notice], inline: true) %>.</span>
       <a href="#" id="close-notice-link">close</a>
     </div>
 

app/views/posts/partials/index/_posts.html.erb

@@ -6,11 +6,15 @@
   <% if post_set.hidden_posts.present? %>
     <div class="tn hidden-posts-notice">
       <% if post_set.banned_posts.present? %>
-        <%= post_set.banned_posts.size %> post(s) were removed from this page at the artist's request (<%= link_to "learn more", wiki_pages_path(title: "banned_artist") %>).
+        <%= post_set.banned_posts.size %> post(s) were removed from this page at the artist's request (<%= link_to "learn more", wiki_pages_path(title: "banned_artist") %>).<br>
       <% end %>
 
       <% if post_set.censored_posts.present? %>
-        <%= post_set.censored_posts.size %> post(s) on this page require a <%= link_to "Gold account", new_user_upgrade_path %> to view (<%= link_to "learn more", wiki_pages_path(title: "help:censored_tags") %>).
+        <%= post_set.censored_posts.size %> post(s) on this page require a <%= link_to "Gold account", new_user_upgrade_path %> to view (<%= link_to "learn more", wiki_pages_path(title: "help:censored_tags") %>).<br>
+      <% end %>
+
+      <% if post_set.safe_posts.present? %>
+        <%= post_set.safe_posts.size %> post(s) on this page were hidden by safe mode (<%= Danbooru.config.app_name %>). Go to <%= link_to "Danbooru", "http://danbooru.donmai.us" %> or disable safe mode to view (<%= link_to "learn more", wiki_pages_path(title: "help:user_settings") %>).<br>
       <% end %>
     </div>
   <% end %>

app/views/posts/show.html.erb

@@ -174,7 +174,7 @@
   <meta property="og:title" content="<%= @post.presenter.humanized_essential_tag_string %> - <%= Danbooru.config.app_name %>">
 
   <% if @post.visible? %>
-    <meta property="og:image" content="http://<%= Danbooru.config.hostname %><%= @post.large_file_url %>">
+    <meta property="og:image" content="<%= @post.open_graph_image_url %>">
   <% end %>
 
   <% if Danbooru.config.enable_post_search_counts %>
@@ -189,7 +189,7 @@
     <meta name="twitter:description" content="<%= @post.presenter.humanized_tag_string %> - <%= Danbooru.config.app_name %>">
 
     <% if @post.visible? %>
-      <meta name="twitter:image" content="http://<%= Danbooru.config.hostname %><%= @post.large_file_url %>">
+      <meta name="twitter:image" content="<%= @post.open_graph_image_url %>">
     <% end %>
   <% end %>
 

config/application.rb

@@ -30,11 +30,6 @@ module Danbooru
       config.x.git_hash = nil
     end
 
-    if ENV["DANBOORU_RAKISMET_KEY"]
-      config.rakismet.key = ENV["DANBOORU_RAKISMET_KEY"]
-      config.rakismet.url = ENV["DANBOORU_RAKISMET_URL"]
-    end
-
     config.after_initialize do
       Rails.application.routes.default_url_options = {
         host: Danbooru.config.hostname,

config/danbooru_default_config.rb

@@ -631,6 +631,13 @@ module Danbooru
 
     def recaptcha_secret_key
     end
+    
+    # Akismet API key. Used for Dmail spam detection. http://akismet.com/signup/
+    def rakismet_key
+    end
+
+    def rakismet_url
+    end
   end
 
   class EnvironmentConfiguration

config/initializers/active_record_extensions.rb

@@ -1,46 +0,0 @@
-module Danbooru
-  module Extensions
-    module ActiveRecord
-      extend ActiveSupport::Concern
-
-      module ClassMethods
-        def without_timeout
-          connection.execute("SET STATEMENT_TIMEOUT = 0") unless Rails.env == "test"
-          yield
-        ensure
-          connection.execute("SET STATEMENT_TIMEOUT = #{CurrentUser.user.try(:statement_timeout) || 3_000}") unless Rails.env == "test"
-        end
-
-        def with_timeout(n, default_value = nil, new_relic_params = {})
-          connection.execute("SET STATEMENT_TIMEOUT = #{n}") unless Rails.env == "test"
-          yield
-        rescue ::ActiveRecord::StatementInvalid => x
-          if Rails.env.production?
-            NewRelic::Agent.notice_error(x, :custom_params => new_relic_params.merge(:user_id => CurrentUser.id, :user_ip_addr => CurrentUser.ip_addr))
-          end
-          return default_value
-        ensure
-          connection.execute("SET STATEMENT_TIMEOUT = #{CurrentUser.user.try(:statement_timeout) || 3_000}") unless Rails.env == "test"
-        end
-      end
-
-      %w(execute select_value select_values select_all).each do |method_name|
-        define_method("#{method_name}_sql") do |sql, *params|
-          self.class.connection.__send__(method_name, self.class.sanitize_sql_array([sql, *params]))
-        end
-
-        self.class.__send__(:define_method, "#{method_name}_sql") do |sql, *params|
-          connection.__send__(method_name, sanitize_sql_array([sql, *params]))
-        end
-      end
-    end
-  end
-end
-
-class ActiveRecord::Base
-  class << self
-    public :sanitize_sql_array
-  end
-
-  include Danbooru::Extensions::ActiveRecord
-end

config/initializers/postgres_extensions.rb

@@ -1,5 +0,0 @@
-module PostgresExtensions
-  def columns(*params)
-    super.reject {|x| x.sql_type == "tsvector"}
-  end
-end

config/initializers/rakismet.rb

@@ -0,0 +1,2 @@
+Rails.application.config.rakismet.key = Danbooru.config.rakismet_key
+Rails.application.config.rakismet.url = Danbooru.config.rakismet_url

db/migrate/20170319000519_add_created_at_index_to_versions.rb

@@ -1,6 +1,6 @@
 class AddCreatedAtIndexToVersions < ActiveRecord::Migration
   def change
-    ActiveRecord::Base.without_timeout do
+    ApplicationRecord.without_timeout do
       add_index :note_versions, :created_at
       add_index :artist_versions, :created_at
       add_index :wiki_page_versions, :created_at

db/migrate/20170330230231_add_forum_post_id_to_tag_requests.rb

@@ -1,6 +1,6 @@
 class AddForumPostIdToTagRequests < ActiveRecord::Migration
   def change
-  	ActiveRecord::Base.without_timeout do
+  	ApplicationRecord.without_timeout do
   		add_column :tag_aliases, :forum_post_id, :integer
   		add_column :tag_implications, :forum_post_id, :integer
   		add_column :bulk_update_requests, :forum_post_id, :integer

db/migrate/20170414005856_add_unique_name_constraint_to_users.rb

@@ -1,6 +1,6 @@
 class AddUniqueNameConstraintToUsers < ActiveRecord::Migration
   def up
-  	ActiveRecord::Base.without_timeout do
+  	User.without_timeout do
 	  	remove_index :users, :name
   		execute "create unique index index_users_on_name on users(lower(name))"
   	end

db/migrate/20171127195124_add_trigram_index_to_tags.rb

@@ -0,0 +1,9 @@
+class AddTrigramIndexToTags < ActiveRecord::Migration
+  def up
+    execute "create index index_tags_on_name_trgm on tags using gin (name gin_trgm_ops)"
+  end
+
+  def down
+    execute "drop index index_tags_on_name_trgm"
+  end
+end

db/structure.sql

@@ -7008,6 +7008,13 @@ CREATE UNIQUE INDEX index_tags_on_name ON tags USING btree (name);
 CREATE INDEX index_tags_on_name_pattern ON tags USING btree (name text_pattern_ops);
 
 
+--
+-- Name: index_tags_on_name_trgm; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_tags_on_name_trgm ON tags USING gin (name gin_trgm_ops);
+
+
 --
 -- Name: index_token_buckets_on_user_id; Type: INDEX; Schema: public; Owner: -
 --
@@ -7521,3 +7528,5 @@ INSERT INTO schema_migrations (version) VALUES ('20170914200122');
 
 INSERT INTO schema_migrations (version) VALUES ('20171106075030');
 
+INSERT INTO schema_migrations (version) VALUES ('20171127195124');
+

lib/danbooru/paginator.rb

@@ -3,4 +3,4 @@ require "danbooru/paginator/numbered_collection_extension"
 require "danbooru/paginator/sequential_collection_extension"
 require "danbooru/paginator/pagination_error"
 
-ActiveRecord::Base.__send__(:include, Danbooru::Paginator::ActiveRecordExtension)
+ApplicationRecord.__send__(:include, Danbooru::Paginator::ActiveRecordExtension)

script/fixes/046_fix_nicovideo_artist_urls.rb

@@ -2,7 +2,7 @@
 
 require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'config', 'environment'))
 
-ActiveRecord::Base.without_timeout do
+ArtistUrl.without_timeout do
   ArtistUrl.where("normalized_url like ?", "\%nicovideo\%").find_each do |url|
     before = url.normalized_url
     url.normalize

test/unit/comment_test.rb

@@ -244,6 +244,11 @@ class CommentTest < ActiveSupport::TestCase
             @comment.update_attributes({:body => "nope"}, :as => :moderator)
           end
         end
+
+        should "credit the moderator as the updater" do
+          @comment.update({ body: "test" }, as: :moderator)
+          assert_equal(@mod.id, @comment.updater_id)
+        end
       end
 
       context "that is below the score threshold" do

test/unit/moderator/tag_batch_change_test.rb

@@ -34,8 +34,21 @@ module Moderator
         ss = FactoryGirl.create(:saved_search, :user => @user, :query => "123 ... 456")
         tag_batch_change = TagBatchChange.new("...", "bbb", @user.id, "127.0.0.1")
         tag_batch_change.perform
-        ss.reload
+
-        assert_equal(%w(123 456 bbb), ss.query.scan(/\S+/).sort)
+        assert_equal("123 456 bbb", ss.reload.normalized_query)
+      end
+
+      should "move only saved searches that match the mass update exactly" do
+        ss = FactoryGirl.create(:saved_search, :user => @user, :query => "123 ... 456")
+        tag_batch_change = TagBatchChange.new("1", "bbb", @user.id, "127.0.0.1")
+        tag_batch_change.perform
+
+        assert_equal("... 123 456", ss.reload.normalized_query, "expected '123' to remain unchanged")
+
+        tag_batch_change = TagBatchChange.new("123 456", "789", @user.id, "127.0.0.1")
+        tag_batch_change.perform
+
+        assert_equal("... 789", ss.reload.normalized_query, "expected '123 456' to be changed to '789'")
       end
 
       should "raise an error if there is no predicate" do

test/unit/post_test.rb

@@ -353,6 +353,17 @@ class PostTest < ActiveSupport::TestCase
           p1.reload
           assert(p1.has_children?, "Parent should have children")
         end
+
+        should "clear the has_active_children flag when the 'move favorites' option is set" do
+          user = FactoryGirl.create(:gold_user)
+          p1 = FactoryGirl.create(:post)
+          c1 = FactoryGirl.create(:post, :parent_id => p1.id)
+          c1.add_favorite!(user)
+
+          assert_equal(true, p1.reload.has_active_children?)
+          c1.delete!("test", :move_favorites => true)
+          assert_equal(false, p1.reload.has_active_children?)
+        end
       end
 
       context "one child" do
@@ -779,6 +790,14 @@ class PostTest < ActiveSupport::TestCase
             assert(Tag.where(name: "someone_(cosplay)", category: 4).exists?, "expected 'someone_(cosplay)' tag to be created as character")
             assert(Tag.where(name: "someone", category: 4).exists?, "expected 'someone' tag to be created")
           end
+
+          should "apply aliases when the character tag is added" do
+            FactoryGirl.create(:tag_alias, antecedent_name: "jim", consequent_name: "james")
+            @post.add_tag("jim_(cosplay)")
+            @post.save
+
+            assert(@post.has_tag?("james"), "expected 'jim' to be aliased to 'james'")
+          end
         end
 
         context "for a parent" do
@@ -816,6 +835,25 @@ class PostTest < ActiveSupport::TestCase
           end
         end
 
+        context "for a favgroup" do
+          setup do
+            @favgroup = FactoryGirl.create(:favorite_group, creator: @user)
+            @post = FactoryGirl.create(:post, :tag_string => "aaa favgroup:#{@favgroup.id}")
+          end
+
+          should "add the post to the favgroup" do
+            assert_equal(1, @favgroup.reload.post_count)
+            assert_equal(true, !!@favgroup.contains?(@post.id))
+          end
+
+          should "remove the post from the favgroup" do
+            @post.update(:tag_string => "-favgroup:#{@favgroup.id}")
+
+            assert_equal(0, @favgroup.reload.post_count)
+            assert_equal(false, !!@favgroup.contains?(@post.id))
+          end
+        end
+
         context "for a pool" do
           setup do
             mock_pool_archive_service!
@@ -1549,6 +1587,42 @@ class PostTest < ActiveSupport::TestCase
           assert_equal("http://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", @post.normalized_source)
         end
       end
+
+      context "when validating tags" do
+        should "warn when creating a new general tag" do
+          @post.add_tag("tag")
+          @post.save
+
+          assert_match(/Created 1 new tag: \[\[tag\]\]/, @post.warnings.full_messages.join)
+        end
+
+        should "warn when adding an artist tag without an artist entry" do
+          @post.add_tag("artist:bkub")
+          @post.save
+
+          assert_match(/Artist \[\[bkub\]\] requires an artist entry./, @post.warnings.full_messages.join)
+        end
+
+        should "warn when a tag removal failed due to implications or automatic tags" do
+          ti = FactoryGirl.create(:tag_implication, antecedent_name: "cat", consequent_name: "animal")
+          @post.reload
+          @post.update(old_tag_string: @post.tag_string, tag_string: "chen_(cosplay) chen cosplay cat animal")
+          @post.reload
+          @post.update(old_tag_string: @post.tag_string, tag_string: "chen_(cosplay) chen cosplay cat -cosplay")
+
+          assert_match(/\[\[animal\]\] and \[\[cosplay\]\] could not be removed./, @post.warnings.full_messages.join)
+        end
+
+        should "warn when a post from a known source is missing an artist tag" do
+          post = FactoryGirl.build(:post, source: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=65985331")
+          post.save
+          assert_match(/Artist tag is required/, post.warnings.full_messages.join)
+        end
+
+        should "warn when missing a copyright tag" do
+          assert_match(/Copyright tag is required/, @post.warnings.full_messages.join)
+        end
+      end
     end
   end
 

test/unit/post_view_count_service_test.rb

@@ -49,6 +49,7 @@ class PostViewCountServiceTest < ActiveSupport::TestCase
 
     context "failure" do
       setup do
+        @date = "2000-01-01"
         stub_request(:get, "localhost:1234/post_views/rank").with(query: {"date" => @date}).to_return(body: "", status: 400)
       end