jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into fix-recaptcha

Browse Source
This commit is contained in:
Albert Yi
2017-12-13 14:33:39 -08:00
committed by GitHub
parent 8d8a2f9c1e 5f1da19f50
commit c3f1c786e8
36 changed files with 396 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/assets/javascripts/autocomplete.js
View File

@@ -244,7 +244,7 @@
$.ajax({
url: "/tags/autocomplete.json",
data: {
"search[name_matches]": term + "*"
"search[name_matches]": term
},
method: "get",
success: function(data) {

4
app/controllers/posts_controller.rb
View File

@@ -110,6 +110,10 @@ private
def respond_with_post_after_update(post)
respond_with(post) do |format|
format.html do
if post.warnings.any?
flash[:notice] = post.warnings.full_messages.join(".\n \n")
end
if post.errors.any?
@error_message = post.errors.full_messages.join("; ")
render :template => "static/error", :status => 500

7
app/controllers/uploads_controller.rb
View File

@@ -53,7 +53,12 @@ class UploadsController < ApplicationController
def create
@upload = Upload.create(params[:upload].merge(:server => Socket.gethostname))
@upload.process! if @upload.errors.empty?
if @upload.errors.empty?
post = @upload.process!
flash[:notice] = post.warnings.full_messages.join(".\n \n") if post.present? && post.warnings.any?
end
save_recent_tags
respond_with(@upload)
end

2
app/logical/bulk_revert.rb
View File

@@ -10,7 +10,7 @@ class BulkRevert
ModAction.log("Processed bulk revert for #{constraints.inspect} by #{creator.name}")
CurrentUser.scoped(creator) do
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
find_post_versions.order("updated_at, id").each do |version|
version.undo!
end

18
app/logical/moderator/dashboard/report.rb
View File

@@ -9,37 +9,37 @@ module Moderator
end
def artists
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::Artist.all(min_date, max_level)
end
end
def comments
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::Comment.all(min_date, max_level)
end
end
def mod_actions
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::ModAction.all
end
end
def notes
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::Note.all(min_date, max_level)
end
end
def appeals
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::PostAppeal.all(min_date)
end
end
def flags
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::PostFlag.all(min_date)
end
end
@@ -49,19 +49,19 @@ module Moderator
end
def posts
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::Upload.all(min_date, max_level)
end
end
def user_feedbacks
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::UserFeedback.all
end
end
def wiki_pages
ActiveRecord::Base.without_timeout do
ApplicationRecord.without_timeout do
Queries::WikiPage.all(min_date, max_level)
end
end

9
app/logical/moderator/tag_batch_change.rb
View File

@@ -18,11 +18,12 @@ module Moderator
post.update_attributes(:tag_string => tags)
end
tags = Tag.scan_tags(antecedent, :strip_metatags => true)
conds = tags.map {|x| "query like ?"}.join(" AND ")
conds = [conds, *tags.map {|x| "%#{x.to_escaped_for_sql_like}%"}]
if SavedSearch.enabled?
SavedSearch.where(*conds).find_each do |ss|
tags = Tag.scan_tags(antecedent, :strip_metatags => true)
# https://www.postgresql.org/docs/current/static/functions-array.html
saved_searches = SavedSearch.where("string_to_array(query, ' ') @> ARRAY[?]", tags)
saved_searches.find_each do |ss|
ss.query = (ss.query.split - tags + [consequent]).uniq.join(" ")
ss.save
end

8
app/logical/post_sets/post.rb
View File

@@ -87,11 +87,15 @@ module PostSets
end
def banned_posts
posts.select(&:is_banned?)
posts.select { |p| p.banblocked? }
end
def censored_posts
hidden_posts - banned_posts
posts.select { |p| p.levelblocked? && !p.banblocked? }
end
def safe_posts
posts.select { |p| p.safeblocked? && !p.levelblocked? && !p.banblocked? }
end
def use_sequential_paginator?

45
app/models/application_record.rb
View File

@@ -43,5 +43,50 @@ class ApplicationRecord < ActiveRecord::Base
end
end
concerning :ActiveRecordExtensions do
class_methods do
def without_timeout
connection.execute("SET STATEMENT_TIMEOUT = 0") unless Rails.env == "test"
yield
ensure
connection.execute("SET STATEMENT_TIMEOUT = #{CurrentUser.user.try(:statement_timeout) || 3_000}") unless Rails.env == "test"
end
def with_timeout(n, default_value = nil, new_relic_params = {})
connection.execute("SET STATEMENT_TIMEOUT = #{n}") unless Rails.env == "test"
yield
rescue ::ActiveRecord::StatementInvalid => x
if Rails.env.production?
NewRelic::Agent.notice_error(x, :custom_params => new_relic_params.merge(:user_id => CurrentUser.id, :user_ip_addr => CurrentUser.ip_addr))
end
return default_value
ensure
connection.execute("SET STATEMENT_TIMEOUT = #{CurrentUser.user.try(:statement_timeout) || 3_000}") unless Rails.env == "test"
end
end
%w(execute select_value select_values select_all).each do |method_name|
define_method("#{method_name}_sql") do |sql, *params|
self.class.connection.__send__(method_name, self.class.send(:sanitize_sql_array, [sql, *params]))
end
self.class.__send__(:define_method, "#{method_name}_sql") do |sql, *params|
connection.__send__(method_name, send(:sanitize_sql_array, [sql, *params]))
end
end
end
concerning :PostgresExtensions do
class_methods do
def columns(*params)
super.reject {|x| x.sql_type == "tsvector"}
end
end
end
def warnings
@warnings ||= ActiveModel::Errors.new(self)
end
include ApiMethods
end

6
app/models/comment.rb
View File

@@ -182,8 +182,8 @@ class Comment < ApplicationRecord
end
def initialize_updater
self.updater_id ||= CurrentUser.user.id
self.updater_ip_addr ||= CurrentUser.ip_addr
self.updater_id = CurrentUser.user.id
self.updater_ip_addr = CurrentUser.ip_addr
end
def creator_name
@@ -260,5 +260,3 @@ class Comment < ApplicationRecord
DText.quote(body, creator_name)
end
end
Comment.connection.extend(PostgresExtensions)

13
app/models/dmail.rb
View File

@@ -30,6 +30,12 @@ class Dmail < ApplicationRecord
def creator_ip_addr_str
creator_ip_addr.to_s
end
def spam?(sender = CurrentUser.user)
return false if Danbooru.config.rakismet_key.blank?
return false if sender.is_gold?
super()
end
end
module AddressMethods
@@ -52,12 +58,7 @@ class Dmail < ApplicationRecord
def initialize_attributes
self.from_id ||= CurrentUser.id
self.creator_ip_addr ||= CurrentUser.ip_addr
if CurrentUser.is_gold?
self.is_spam = false
else
self.is_spam = spam?
end
true
self.is_spam = spam?(CurrentUser.user)
end
end

20
app/models/favorite_group.rb
View File

@@ -173,11 +173,13 @@ class FavoriteGroup < ApplicationRecord
end
def add!(post_id)
post_id = post_id.id if post_id.is_a?(Post)
return if contains?(post_id)
with_lock do
post_id = post_id.id if post_id.is_a?(Post)
return if contains?(post_id)
clear_post_id_array
update_attributes(:post_ids => add_number_to_string(post_id, post_ids))
clear_post_id_array
update_attributes(:post_ids => add_number_to_string(post_id, post_ids))
end
end
def self.purge_post(post_id)
@@ -188,11 +190,13 @@ class FavoriteGroup < ApplicationRecord
end
def remove!(post_id)
post_id = post_id.id if post_id.is_a?(Post)
return unless contains?(post_id)
with_lock do
post_id = post_id.id if post_id.is_a?(Post)
return unless contains?(post_id)
clear_post_id_array
update_attributes(:post_ids => remove_number_from_string(post_id, post_ids))
clear_post_id_array
update_attributes(:post_ids => remove_number_from_string(post_id, post_ids))
end
end
def add_number_to_string(number, string)

125
app/models/post.rb
View File

@@ -18,6 +18,10 @@ class Post < ApplicationRecord
validates_uniqueness_of :md5, :on => :create
validates_inclusion_of :rating, in: %w(s q e), message: "rating must be s, q, or e"
validate :tag_names_are_valid
validate :added_tags_are_valid
validate :removed_tags_are_valid
validate :has_artist_tag
validate :has_copyright_tag
validate :post_is_not_its_own_parent
validate :updater_can_change_rating
before_save :update_tag_post_counts
@@ -196,6 +200,14 @@ class Post < ApplicationRecord
"http://#{Danbooru.config.hostname}#{preview_file_url}"
end
def open_graph_image_url
if is_image? && has_large?
"http://#{Danbooru.config.hostname}#{large_file_url}"
else
complete_preview_file_url
end
end
def file_url_for(user)
if CurrentUser.mobile_mode?
large_file_url
@@ -592,6 +604,18 @@ class Post < ApplicationRecord
@tag_array_was ||= Tag.scan_tags(tag_string_was)
end
def tags
Tag.where(name: tag_array)
end
def tags_was
Tag.where(name: tag_array_was)
end
def added_tags
tags - tags_was
end
def decrement_tag_post_counts
Tag.where(:name => tag_array).update_all("post_count = post_count - 1") if tag_array.any?
end
@@ -633,12 +657,18 @@ class Post < ApplicationRecord
end
def merge_old_changes
@removed_tags = []
if old_tag_string
# If someone else committed changes to this post before we did,
# then try to merge the tag changes together.
current_tags = tag_array_was()
new_tags = tag_array()
old_tags = Tag.scan_tags(old_tag_string)
kept_tags = current_tags & new_tags
@removed_tags = old_tags - kept_tags
set_tag_string(((current_tags + new_tags) - old_tags + (current_tags & new_tags)).uniq.sort.join(" "))
end
@@ -687,10 +717,10 @@ class Post < ApplicationRecord
end
def remove_negated_tags(tags)
negated_tags, tags = tags.partition {|x| x =~ /\A-/i}
negated_tags = negated_tags.map {|x| x[1..-1]}
negated_tags = TagAlias.to_aliased(negated_tags)
return tags - negated_tags
@negated_tags, tags = tags.partition {|x| x =~ /\A-/i}
@negated_tags = @negated_tags.map {|x| x[1..-1]}
@negated_tags = TagAlias.to_aliased(@negated_tags)
return tags - @negated_tags
end
def add_automatic_tags(tags)
@@ -1369,18 +1399,15 @@ class Post < ApplicationRecord
Post.transaction do
flag!(reason, is_deletion: true)
self.is_deleted = true
self.is_pending = false
self.is_flagged = false
self.is_banned = true if options[:ban] || has_tag?("banned_artist")
update_columns(
:is_deleted => is_deleted,
:is_pending => is_pending,
:is_flagged => is_flagged,
:is_banned => is_banned
)
update({
is_deleted: true,
is_pending: false,
is_flagged: false,
is_banned: is_banned || options[:ban] || has_tag?("banned_artist")
}, without_protection: true)
# XXX This must happen *after* the `is_deleted` flag is set to true (issue #3419).
give_favorites_to_parent if options[:move_favorites]
update_parent_on_save
unless options[:without_mod_action]
ModAction.log("deleted post ##{id}, reason: #{reason}")
@@ -1708,6 +1735,53 @@ class Post < ApplicationRecord
end
end
end
def added_tags_are_valid
new_tags = added_tags.select { |t| t.post_count <= 1 }
new_general_tags = new_tags.select { |t| t.category == Tag.categories.general }
new_artist_tags = new_tags.select { |t| t.category == Tag.categories.artist }
if new_general_tags.present?
n = new_general_tags.size
tag_wiki_links = new_general_tags.map { |tag| "[[#{tag.name}]]" }
self.warnings[:base] << "Created #{n} new #{n == 1 ? "tag" : "tags"}: #{tag_wiki_links.join(", ")}"
end
new_artist_tags.each do |tag|
if tag.artist.blank?
self.warnings[:base] << "Artist [[#{tag.name}]] requires an artist entry. \"Create new artist entry\":[/artists/new?name=#{CGI::escape(tag.name)}]"
end
end
end
def removed_tags_are_valid
attempted_removed_tags = @removed_tags + @negated_tags
unremoved_tags = tag_array & attempted_removed_tags
if unremoved_tags.present?
unremoved_tags_list = unremoved_tags.map { |t| "[[#{t}]]" }.to_sentence
self.warnings[:base] << "#{unremoved_tags_list} could not be removed. Check for implications and try again"
end
end
def has_artist_tag
return if !new_record?
return if source !~ %r!\Ahttps?://!
return if has_tag?("artist_request") || has_tag?("official_art")
return if tags.any? { |t| t.category == Tag.categories.artist }
site = Sources::Site.new(source)
self.warnings[:base] << "Artist tag is required. Create a new tag with [[artist:<artist_name>]]. Ask on the forum if you need naming help"
rescue Sources::Site::NoStrategyError => e
# unrecognized source; do nothing.
end
def has_copyright_tag
return if !new_record?
return if has_tag?("copyright_request") || tags.any? { |t| t.category == Tag.categories.copyright }
self.warnings[:base] << "Copyright tag is required. Consider adding [[copyright request]] or [[original]]"
end
end
include FileMethods
@@ -1738,11 +1812,22 @@ class Post < ApplicationRecord
)
has_bit_flags BOOLEAN_ATTRIBUTES
def safeblocked?
CurrentUser.safe_mode? && (rating != "s" || has_tag?("toddlercon|toddler|diaper|tentacle|rape|bestiality|beastiality|lolita|loli|nude|shota|pussy|penis"))
end
def levelblocked?
!Danbooru.config.can_user_see_post?(CurrentUser.user, self)
end
def banblocked?
is_banned? && !CurrentUser.is_gold?
end
def visible?
return false if !Danbooru.config.can_user_see_post?(CurrentUser.user, self)
return false if CurrentUser.safe_mode? && rating != "s"
return false if CurrentUser.safe_mode? && has_tag?("toddlercon|toddler|diaper|tentacle|rape|bestiality|beastiality|lolita|loli|nude|shota|pussy|penis")
return false if is_banned? && !CurrentUser.is_gold?
return false if safeblocked?
return false if levelblocked?
return false if banblocked?
return true
end
@@ -1798,5 +1883,3 @@ class Post < ApplicationRecord
ret
end
end
Post.connection.extend(PostgresExtensions)

39
app/models/tag.rb
View File

@@ -6,6 +6,7 @@ class Tag < ApplicationRecord
attr_accessible :category, :as => [:moderator, :gold, :platinum, :member, :anonymous, :default, :builder, :admin]
attr_accessible :is_locked, :as => [:moderator, :admin]
has_one :wiki_page, :foreign_key => "title", :primary_key => "name"
has_one :artist, :foreign_key => "name", :primary_key => "name"
has_one :antecedent_alias, lambda {active}, :class_name => "TagAlias", :foreign_key => "antecedent_name", :primary_key => "name"
has_many :consequent_aliases, lambda {active}, :class_name => "TagAlias", :foreign_key => "consequent_name", :primary_key => "name"
has_many :antecedent_implications, lambda {active}, :class_name => "TagImplication", :foreign_key => "antecedent_name", :primary_key => "name"
@@ -816,6 +817,18 @@ class Tag < ApplicationRecord
where("tags.post_count > 0")
end
# ref: https://www.postgresql.org/docs/current/static/pgtrgm.html#idm46428634524336
def order_similarity(name)
# trunc(3 * sim) reduces the similarity score from a range of 0.0 -> 1.0 to just 0, 1, or 2.
# This groups tags first by approximate similarity, then by largest tags within groups of similar tags.
order("trunc(3 * similarity(name, #{sanitize(name)})) DESC", "post_count DESC", "name DESC")
end
# ref: https://www.postgresql.org/docs/current/static/pgtrgm.html#idm46428634524336
def fuzzy_name_matches(name)
where("tags.name % ?", name)
end
def name_matches(name)
where("tags.name LIKE ? ESCAPE E'\\\\'", normalize_name(name).to_escaped_for_sql_like)
end
@@ -828,6 +841,10 @@ class Tag < ApplicationRecord
q = where("true")
params = {} if params.blank?
if params[:fuzzy_name_matches].present?
q = q.fuzzy_name_matches(params[:fuzzy_name_matches])
end
if params[:name_matches].present?
q = q.name_matches(params[:name_matches])
end
@@ -864,6 +881,8 @@ class Tag < ApplicationRecord
q = q.reorder("id desc")
when "count"
q = q.reorder("post_count desc")
when "similarity"
q = q.order_similarity(params[:fuzzy_name_matches]) if params[:fuzzy_name_matches].present?
else
q = q.reorder("id desc")
end
@@ -872,21 +891,29 @@ class Tag < ApplicationRecord
end
def names_matches_with_aliases(name)
query1 = Tag.select("tags.name, tags.post_count, tags.category, null AS antecedent_name")
.search(:name_matches => name, :order => "count").limit(10)
name = normalize_name(name)
wildcard_name = name + '*'
query1 = Tag.select("tags.name, tags.post_count, tags.category, null AS antecedent_name")
.search(:name_matches => wildcard_name, :order => "count").limit(10)
name = name.mb_chars.downcase.to_escaped_for_sql_like
query2 = TagAlias.select("tags.name, tags.post_count, tags.category, tag_aliases.antecedent_name")
.joins("INNER JOIN tags ON tags.name = tag_aliases.consequent_name")
.where("tag_aliases.antecedent_name LIKE ? ESCAPE E'\\\\'", name)
.where("tag_aliases.antecedent_name LIKE ? ESCAPE E'\\\\'", wildcard_name.to_escaped_for_sql_like)
.active
.where("tags.name NOT LIKE ? ESCAPE E'\\\\'", name)
.where("tags.name NOT LIKE ? ESCAPE E'\\\\'", wildcard_name.to_escaped_for_sql_like)
.where("tag_aliases.post_count > 0")
.order("tag_aliases.post_count desc")
.limit(20) # Get 20 records even though only 10 will be displayed in case some duplicates get filtered out.
sql_query = "((#{query1.to_sql}) UNION ALL (#{query2.to_sql})) AS unioned_query"
Tag.select("DISTINCT ON (name, post_count) *").from(sql_query).order("post_count desc").limit(10)
tags = Tag.select("DISTINCT ON (name, post_count) *").from(sql_query).order("post_count desc").limit(10)
if tags.empty?
tags = Tag.fuzzy_name_matches(name).order_similarity(name).nonempty.limit(10)
end
tags
end
end

2
app/models/tag_implication.rb
View File

@@ -21,7 +21,7 @@ class TagImplication < TagRelationship
end
def automatic_tags_for(names)
tags = names.grep(/\A(.+)_\(cosplay\)\Z/) { "char:#{$1}" }
tags = names.grep(/\A(.+)_\(cosplay\)\Z/) { "char:#{TagAlias.to_aliased([$1]).first}" }
tags << "cosplay" if tags.present?
tags.uniq
end

6
app/models/upload.rb
View File

@@ -148,6 +148,8 @@ class Upload < ApplicationRecord
else
update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
end
post
end
def process!(force = false)
@@ -155,7 +157,7 @@ class Upload < ApplicationRecord
return if !force && status =~ /processing|completed|error/
process_upload
create_post_from_upload
post = create_post_from_upload
rescue Timeout::Error, Net::HTTP::Persistent::Error => x
if @tries > 3
@@ -164,9 +166,11 @@ class Upload < ApplicationRecord
@tries += 1
retry
end
nil
rescue Exception => x
update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
nil
ensure
delete_temp_file

15
app/presenters/post_presenter.rb
View File

@@ -155,10 +155,19 @@ class PostPresenter < Presenter
categorized_tag_groups.flatten.slice(0, 25).join(", ").tr("_", " ")
end
def safe_mode_message(template)
html = ["This image is unavailable on safe mode (#{Danbooru.config.app_name}). Go to "]
html << template.link_to("Danbooru", "http://danbooru.donmai.us")
html << " or disable safe mode to view ("
html << template.link_to("learn more", template.wiki_pages_path(title: "help:user_settings"))
html << ")."
html.join.html_safe
end
def image_html(template)
return template.content_tag("p", "The artist requested removal of this image") if @post.is_banned? && !CurrentUser.user.is_gold?
return template.content_tag("p", template.link_to("You need a gold account to see this image.", template.new_user_upgrade_path)) if !Danbooru.config.can_user_see_post?(CurrentUser.user, @post)
return template.content_tag("p", "This image is unavailable") if !@post.visible?
return template.content_tag("p", "The artist requested removal of this image") if @post.banblocked?
return template.content_tag("p", template.link_to("You need a gold account to see this image.", template.new_user_upgrade_path)) if @post.levelblocked?
return template.content_tag("p", safe_mode_message(template)) if @post.safeblocked?
if @post.is_flash?
template.render("posts/partials/show/flash", :post => @post)

2
app/views/layouts/default.html.erb
View File

@@ -112,7 +112,7 @@
<% end %>
<div class="ui-corner-all ui-state-highlight" id="notice" style="<%= "display: none;" unless flash[:notice] %>">
<span><%= flash[:notice] %></span>
<span><%= format_text(flash[:notice], inline: true) %>.</span>
<a href="#" id="close-notice-link">close</a>
</div>

8
app/views/posts/partials/index/_posts.html.erb
View File

@@ -6,11 +6,15 @@
<% if post_set.hidden_posts.present? %>
<div class="tn hidden-posts-notice">
<% if post_set.banned_posts.present? %>
<%= post_set.banned_posts.size %> post(s) were removed from this page at the artist's request (<%= link_to "learn more", wiki_pages_path(title: "banned_artist") %>).
<%= post_set.banned_posts.size %> post(s) were removed from this page at the artist's request (<%= link_to "learn more", wiki_pages_path(title: "banned_artist") %>).<br>
<% end %>
<% if post_set.censored_posts.present? %>
<%= post_set.censored_posts.size %> post(s) on this page require a <%= link_to "Gold account", new_user_upgrade_path %> to view (<%= link_to "learn more", wiki_pages_path(title: "help:censored_tags") %>).
<%= post_set.censored_posts.size %> post(s) on this page require a <%= link_to "Gold account", new_user_upgrade_path %> to view (<%= link_to "learn more", wiki_pages_path(title: "help:censored_tags") %>).<br>
<% end %>
<% if post_set.safe_posts.present? %>
<%= post_set.safe_posts.size %> post(s) on this page were hidden by safe mode (<%= Danbooru.config.app_name %>). Go to <%= link_to "Danbooru", "http://danbooru.donmai.us" %> or disable safe mode to view (<%= link_to "learn more", wiki_pages_path(title: "help:user_settings") %>).<br>
<% end %>
</div>
<% end %>

4
app/views/posts/show.html.erb
View File

@@ -174,7 +174,7 @@
<meta property="og:title" content="<%= @post.presenter.humanized_essential_tag_string %> - <%= Danbooru.config.app_name %>">
<% if @post.visible? %>
<meta property="og:image" content="http://<%= Danbooru.config.hostname %><%= @post.large_file_url %>">
<meta property="og:image" content="<%= @post.open_graph_image_url %>">
<% end %>
<% if Danbooru.config.enable_post_search_counts %>
@@ -189,7 +189,7 @@
<meta name="twitter:description" content="<%= @post.presenter.humanized_tag_string %> - <%= Danbooru.config.app_name %>">
<% if @post.visible? %>
<meta name="twitter:image" content="http://<%= Danbooru.config.hostname %><%= @post.large_file_url %>">
<meta name="twitter:image" content="<%= @post.open_graph_image_url %>">
<% end %>
<% end %>