search: split tag_match into user_tag_match / system_tag_match.

When doing a tag search, we have to be careful about which user we're
running the search as because the results depend on the current user.
Specifically, things like private favorites, private favorite groups,
post votes, saved searches, and flagger names depend on the user's
permissions, and whether non-safe or deleted posts are filtered out
depend on whether the user has safe mode on or the hide deleted posts
setting enabled.

* Refactor internal searches to explicitly state whether they're
  running as the system user (DanbooruBot) or as the current user.
* Explicitly pass in the current user to PostQueryBuilder instead of
  implicitly relying on the CurrentUser global.
* Get rid of CurrentUser.admin_mode? (used to ignore the hide deleted
  post setting) and CurrentUser.without_safe_mode (used to ignore safe
  mode).
* Change the /counts/posts.json endpoint to ignore safe mode and the
  hide deleted posts settings when counting posts.
* Fix searches not correctly overriding the hide deleted posts setting
  when multiple status: metatags were used (e.g. `status:banned status:active`)
* Fix fast_count not respecting the hide deleted posts setting when the
  status:banned metatag was used.
This commit is contained in:
evazion
2020-05-06 22:00:47 -05:00
parent a753ebbea9
commit f38c38f26e
24 changed files with 120 additions and 147 deletions

View File

@@ -82,7 +82,7 @@ class CommentsController < ApplicationController
end end
def index_by_post def index_by_post
@posts = Post.where("last_comment_bumped_at IS NOT NULL").tag_match(params[:tags]).reorder("last_comment_bumped_at DESC NULLS LAST").paginate(params[:page], :limit => 5, :search_count => params[:search]) @posts = Post.where("last_comment_bumped_at IS NOT NULL").user_tag_match(params[:tags]).reorder("last_comment_bumped_at DESC NULLS LAST").paginate(params[:page], :limit => 5, :search_count => params[:search])
if request.format.html? if request.format.html?
@posts = @posts.includes(comments: [:creator]) @posts = @posts.includes(comments: [:creator])

View File

@@ -2,6 +2,6 @@ class CountsController < ApplicationController
respond_to :xml, :json respond_to :xml, :json
def posts def posts
@count = PostQueryBuilder.new(params[:tags]).fast_count(timeout: CurrentUser.statement_timeout, raise_on_timeout: true, skip_cache: params[:skip_cache]) @count = PostQueryBuilder.new(params[:tags], CurrentUser.user).fast_count(timeout: CurrentUser.statement_timeout, raise_on_timeout: true, skip_cache: params[:skip_cache])
end end
end end

View File

@@ -47,11 +47,11 @@ module Explore
end end
def popular_posts(min_date, max_date) def popular_posts(min_date, max_date)
Post.where(created_at: min_date..max_date).tag_match("order:score") Post.where(created_at: min_date..max_date).user_tag_match("order:score")
end end
def curated_posts(min_date, max_date) def curated_posts(min_date, max_date)
Post.where(created_at: min_date..max_date).tag_match("order:curated") Post.where(created_at: min_date..max_date).user_tag_match("order:curated")
end end
end end
end end

View File

@@ -80,7 +80,7 @@ class PostsController < ApplicationController
end end
def random def random
@post = Post.tag_match(params[:tags]).random @post = Post.user_tag_match(params[:tags]).random
raise ActiveRecord::RecordNotFound if @post.nil? raise ActiveRecord::RecordNotFound if @post.nil?
authorize @post authorize @post
respond_with(@post) do |format| respond_with(@post) do |format|

View File

@@ -9,19 +9,17 @@ class TagBatchChangeJob < ApplicationJob
normalized_antecedent = TagAlias.to_aliased(PostQueryBuilder.new(antecedent.mb_chars.downcase).split_query) normalized_antecedent = TagAlias.to_aliased(PostQueryBuilder.new(antecedent.mb_chars.downcase).split_query)
normalized_consequent = TagAlias.to_aliased(PostQueryBuilder.new(consequent.mb_chars.downcase).parse_tag_edit) normalized_consequent = TagAlias.to_aliased(PostQueryBuilder.new(consequent.mb_chars.downcase).parse_tag_edit)
CurrentUser.without_safe_mode do CurrentUser.scoped(updater, updater_ip_addr) do
CurrentUser.scoped(updater, updater_ip_addr) do migrate_posts(normalized_antecedent, normalized_consequent)
migrate_posts(normalized_antecedent, normalized_consequent) migrate_saved_searches(normalized_antecedent, normalized_consequent)
migrate_saved_searches(normalized_antecedent, normalized_consequent) migrate_blacklists(normalized_antecedent, normalized_consequent)
migrate_blacklists(normalized_antecedent, normalized_consequent)
ModAction.log("processed mass update: #{antecedent} -> #{consequent}", :mass_update) ModAction.log("processed mass update: #{antecedent} -> #{consequent}", :mass_update)
end
end end
end end
def migrate_posts(normalized_antecedent, normalized_consequent) def migrate_posts(normalized_antecedent, normalized_consequent)
::Post.tag_match(normalized_antecedent.join(" ")).find_each do |post| ::Post.system_tag_match(normalized_antecedent.join(" ")).find_each do |post|
post.with_lock do post.with_lock do
tags = (post.tag_array - normalized_antecedent + normalized_consequent).join(" ") tags = (post.tag_array - normalized_antecedent + normalized_consequent).join(" ")
post.update(tag_string: tags) post.update(tag_string: tags)

View File

@@ -226,7 +226,7 @@ module Searchable
end end
if params[:post_tags_match].present? if params[:post_tags_match].present?
relation = relation.where(post_id: Post.tag_match(params[:post_tags_match]).reorder(nil)) relation = relation.where(post_id: Post.user_tag_match(params[:post_tags_match]).reorder(nil))
end end
relation relation

View File

@@ -73,20 +73,6 @@ class CurrentUser
RequestStore[:safe_mode] RequestStore[:safe_mode]
end end
def self.admin_mode?
RequestStore[:admin_mode]
end
def self.without_safe_mode
prev = RequestStore[:safe_mode]
RequestStore[:safe_mode] = false
RequestStore[:admin_mode] = true
yield
ensure
RequestStore[:safe_mode] = prev
RequestStore[:admin_mode] = false
end
def self.safe_mode=(safe_mode) def self.safe_mode=(safe_mode)
RequestStore[:safe_mode] = safe_mode RequestStore[:safe_mode] = safe_mode
end end

View File

@@ -53,10 +53,15 @@ class PostQueryBuilder
COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } + COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } +
CATEGORY_COUNT_METATAGS.flat_map { |str| [str, "#{str}_asc"] } CATEGORY_COUNT_METATAGS.flat_map { |str| [str, "#{str}_asc"] }
attr_accessor :query_string attr_reader :query_string, :current_user, :safe_mode, :hide_deleted_posts
alias_method :safe_mode?, :safe_mode
alias_method :hide_deleted_posts?, :hide_deleted_posts
def initialize(query_string) def initialize(query_string, current_user = User.anonymous, safe_mode: false, hide_deleted_posts: false)
@query_string = query_string @query_string = query_string
@current_user = current_user
@safe_mode = safe_mode
@hide_deleted_posts = hide_deleted_posts
end end
def tags_match(tags, relation) def tags_match(tags, relation)
@@ -177,9 +182,9 @@ class PostQueryBuilder
when "noteupdater" when "noteupdater"
user_subquery_matches(NoteVersion.unscoped, value, field: :updater) user_subquery_matches(NoteVersion.unscoped, value, field: :updater)
when "upvoter", "upvote" when "upvoter", "upvote"
user_subquery_matches(PostVote.positive.visible(CurrentUser.user), value, field: :user) user_subquery_matches(PostVote.positive.visible(current_user), value, field: :user)
when "downvoter", "downvote" when "downvoter", "downvote"
user_subquery_matches(PostVote.negative.visible(CurrentUser.user), value, field: :user) user_subquery_matches(PostVote.negative.visible(current_user), value, field: :user)
when *CATEGORY_COUNT_METATAGS when *CATEGORY_COUNT_METATAGS
short_category = name.delete_suffix("tags") short_category = name.delete_suffix("tags")
category = TagCategory.short_name_mapping[short_category] category = TagCategory.short_name_mapping[short_category]
@@ -248,16 +253,16 @@ class PostQueryBuilder
user_subquery_matches(flags, username) do |username| user_subquery_matches(flags, username) do |username|
flagger = User.find_by_name(username) flagger = User.find_by_name(username)
PostFlag.unscoped.creator_matches(flagger, CurrentUser.user) PostFlag.unscoped.creator_matches(flagger, current_user)
end end
end end
def saved_search_matches(label) def saved_search_matches(label)
case label.downcase case label.downcase
when "all" when "all"
Post.where(id: SavedSearch.post_ids_for(CurrentUser.id)) Post.where(id: SavedSearch.post_ids_for(current_user.id))
else else
Post.where(id: SavedSearch.post_ids_for(CurrentUser.id, label: label)) Post.where(id: SavedSearch.post_ids_for(current_user.id, label: label))
end end
end end
@@ -287,8 +292,8 @@ class PostQueryBuilder
def disapproved_matches(query) def disapproved_matches(query)
if query.downcase.in?(PostDisapproval::REASONS) if query.downcase.in?(PostDisapproval::REASONS)
Post.where(disapprovals: PostDisapproval.where(reason: query.downcase)) Post.where(disapprovals: PostDisapproval.where(reason: query.downcase))
elsif User.normalize_name(query) == CurrentUser.user.name elsif User.normalize_name(query) == current_user.name
Post.where(disapprovals: PostDisapproval.where(user: CurrentUser.user)) Post.where(disapprovals: PostDisapproval.where(user: current_user))
else else
Post.none Post.none
end end
@@ -362,20 +367,20 @@ class PostQueryBuilder
def ordfavgroup_matches(query) def ordfavgroup_matches(query)
# XXX unify with FavoriteGroup#posts # XXX unify with FavoriteGroup#posts
favgroup = FavoriteGroup.visible(CurrentUser.user).name_or_id_matches(query, CurrentUser.user) favgroup = FavoriteGroup.visible(current_user).name_or_id_matches(query, current_user)
favgroup_posts = favgroup.joins("CROSS JOIN unnest(favorite_groups.post_ids) WITH ORDINALITY AS row(post_id, favgroup_index)").select(:post_id, :favgroup_index) favgroup_posts = favgroup.joins("CROSS JOIN unnest(favorite_groups.post_ids) WITH ORDINALITY AS row(post_id, favgroup_index)").select(:post_id, :favgroup_index)
Post.joins("JOIN (#{favgroup_posts.to_sql}) favgroup_posts ON favgroup_posts.post_id = posts.id").order("favgroup_posts.favgroup_index ASC") Post.joins("JOIN (#{favgroup_posts.to_sql}) favgroup_posts ON favgroup_posts.post_id = posts.id").order("favgroup_posts.favgroup_index ASC")
end end
def favgroup_matches(query) def favgroup_matches(query)
favgroup = FavoriteGroup.visible(CurrentUser.user).name_or_id_matches(query, CurrentUser.user) favgroup = FavoriteGroup.visible(current_user).name_or_id_matches(query, current_user)
Post.where(id: favgroup.select("unnest(post_ids)")) Post.where(id: favgroup.select("unnest(post_ids)"))
end end
def favorites_include(username) def favorites_include(username)
favuser = User.find_by_name(username) favuser = User.find_by_name(username)
if favuser.present? && Pundit.policy!([CurrentUser.user, nil], favuser).can_see_favorites? if favuser.present? && Pundit.policy!([current_user, nil], favuser).can_see_favorites?
tags_include("fav:#{favuser.id}") tags_include("fav:#{favuser.id}")
else else
Post.none Post.none
@@ -445,10 +450,9 @@ class PostQueryBuilder
relation relation
end end
def hide_deleted_posts? def hide_deleted?
return false if CurrentUser.admin_mode? has_status_metatag = select_metatags(:status).any? { |metatag| metatag.value.downcase.in?(%w[deleted active any all]) }
return false if find_metatag(:status).to_s.downcase.in?(%w[deleted active any all]) hide_deleted_posts? && !has_status_metatag
return CurrentUser.user.hide_deleted_posts?
end end
def build def build
@@ -458,8 +462,8 @@ class PostQueryBuilder
end end
relation = Post.all relation = Post.all
relation = relation.where(rating: 's') if CurrentUser.safe_mode? relation = relation.where(rating: 's') if safe_mode?
relation = relation.undeleted if hide_deleted_posts? relation = relation.undeleted if hide_deleted?
relation = add_joins(relation) relation = add_joins(relation)
relation = metatags_match(metatags, relation) relation = metatags_match(metatags, relation)
relation = tags_match(tags, relation) relation = tags_match(tags, relation)
@@ -813,8 +817,8 @@ class PostQueryBuilder
concerning :CountMethods do concerning :CountMethods do
def fast_count(timeout: 1_000, raise_on_timeout: false, skip_cache: false) def fast_count(timeout: 1_000, raise_on_timeout: false, skip_cache: false)
tags = normalize_query(normalize_aliases: true) tags = normalize_query(normalize_aliases: true)
tags += " rating:s" if CurrentUser.safe_mode? tags += " rating:s" if safe_mode?
tags += " -status:deleted" if CurrentUser.hide_deleted_posts? && !has_metatag?("status") tags += " -status:deleted" if hide_deleted?
tags = tags.strip tags = tags.strip
# Optimize some cases. these are just estimates but at these # Optimize some cases. these are just estimates but at these
@@ -852,7 +856,7 @@ class PostQueryBuilder
def fast_count_search(tags, timeout:, raise_on_timeout:) def fast_count_search(tags, timeout:, raise_on_timeout:)
count = Post.with_timeout(timeout, nil, tags: tags) do count = Post.with_timeout(timeout, nil, tags: tags) do
Post.tag_match(tags).count Post.user_tag_match(tags).count
end end
if count.nil? if count.nil?

View File

@@ -10,9 +10,9 @@ class PostSearchContext
def post_id def post_id
if seq == "prev" if seq == "prev"
Post.tag_match(tags).where("posts.id > ?", id).reorder("posts.id asc").first.try(:id) Post.user_tag_match(tags).where("posts.id > ?", id).reorder("posts.id asc").first.try(:id)
else else
Post.tag_match(tags).where("posts.id < ?", id).reorder("posts.id desc").first.try(:id) Post.user_tag_match(tags).where("posts.id < ?", id).reorder("posts.id desc").first.try(:id)
end end
end end

View File

@@ -4,7 +4,7 @@ module PostSets
attr_reader :page, :random, :post_count, :format, :tag_string, :query attr_reader :page, :random, :post_count, :format, :tag_string, :query
def initialize(tags, page = 1, per_page = nil, random: false, format: "html") def initialize(tags, page = 1, per_page = nil, random: false, format: "html")
@query = PostQueryBuilder.new(tags) @query = PostQueryBuilder.new(tags, CurrentUser.user)
@tag_string = tags @tag_string = tags
@page = page @page = page
@per_page = per_page @per_page = per_page
@@ -92,7 +92,7 @@ module PostSets
def get_random_posts def get_random_posts
per_page.times.inject([]) do |all, x| per_page.times.inject([]) do |all, x|
all << ::Post.tag_match(tag_string).random all << ::Post.user_tag_match(tag_string).random
end.compact.uniq end.compact.uniq
end end
@@ -103,7 +103,7 @@ module PostSets
if is_random? if is_random?
temp = get_random_posts temp = get_random_posts
else else
temp = ::Post.tag_match(tag_string).where("true /* PostSets::Post#posts:2 */").paginate(page, :count => post_count, :limit => per_page) temp = ::Post.user_tag_match(tag_string).where("true /* PostSets::Post#posts:2 */").paginate(page, :count => post_count, :limit => per_page)
end end
end end
end end

View File

@@ -36,7 +36,7 @@ module RecommenderService
posts = posts.where.not(id: post.id) if post posts = posts.where.not(id: post.id) if post
posts = posts.where.not(uploader_id: uploader.id) if uploader posts = posts.where.not(uploader_id: uploader.id) if uploader
posts = posts.where.not(id: favoriter.favorites.select(:post_id)) if favoriter posts = posts.where.not(id: favoriter.favorites.select(:post_id)) if favoriter
posts = posts.where(id: Post.tag_match(tags).reorder(nil).select(:id)) if tags.present? posts = posts.where(id: Post.user_tag_match(tags).reorder(nil).select(:id)) if tags.present?
id_to_score = recs.to_h id_to_score = recs.to_h
recs = posts.map { |post| { score: id_to_score[post.id], post: post } } recs = posts.map { |post| { score: id_to_score[post.id], post: post } }

View File

@@ -1,12 +1,12 @@
module RelatedTagCalculator module RelatedTagCalculator
def self.similar_tags_for_search(tag_query, search_sample_size: 1000, tag_sample_size: 250, category: nil) def self.similar_tags_for_search(tag_query, current_user, search_sample_size: 1000, tag_sample_size: 250, category: nil)
search_count = PostQueryBuilder.new(tag_query).fast_count search_count = PostQueryBuilder.new(tag_query, current_user).fast_count
return [] if search_count.nil? return [] if search_count.nil?
search_sample_size = [search_count, search_sample_size].min search_sample_size = [search_count, search_sample_size].min
return [] if search_sample_size <= 0 return [] if search_sample_size <= 0
tags = frequent_tags_for_search(tag_query, search_sample_size: search_sample_size, category: category).limit(tag_sample_size) tags = frequent_tags_for_search(tag_query, current_user, search_sample_size: search_sample_size, category: category).limit(tag_sample_size)
tags = tags.sort_by do |tag| tags = tags.sort_by do |tag|
# cosine distance(tag1, tag2) = 1 - {{tag1 tag2}} / sqrt({{tag1}} * {{tag2}}) # cosine distance(tag1, tag2) = 1 - {{tag1 tag2}} / sqrt({{tag1}} * {{tag2}})
1 - tag.overlap_count / Math.sqrt(tag.post_count * search_count.to_f) 1 - tag.overlap_count / Math.sqrt(tag.post_count * search_count.to_f)
@@ -15,8 +15,8 @@ module RelatedTagCalculator
tags tags
end end
def self.frequent_tags_for_search(tag_query, search_sample_size: 1000, category: nil) def self.frequent_tags_for_search(tag_query, current_user, search_sample_size: 1000, category: nil)
sample_posts = Post.tag_match(tag_query).reorder(:md5).limit(search_sample_size) sample_posts = Post.user_tag_match(tag_query, current_user).reorder(:md5).limit(search_sample_size)
frequent_tags_for_post_relation(sample_posts, category: category) frequent_tags_for_post_relation(sample_posts, category: category)
end end
@@ -36,12 +36,10 @@ module RelatedTagCalculator
tags_with_counts.sort_by { |tag_name, count| [-count, tag_name] }.map(&:first) tags_with_counts.sort_by { |tag_name, count| [-count, tag_name] }.map(&:first)
end end
def self.cached_similar_tags_for_search(tag_query, max_tags, search_timeout: 2000, cache_timeout: 8.hours) def self.cached_similar_tags_for_search(tag_query, max_tags, current_user, search_timeout: 2000, cache_timeout: 8.hours)
Cache.get("similar_tags:#{tag_query}", cache_timeout, race_condition_ttl: 60.seconds) do Cache.get("similar_tags:#{tag_query}", cache_timeout, race_condition_ttl: 60.seconds) do
ApplicationRecord.with_timeout(search_timeout, []) do ApplicationRecord.with_timeout(search_timeout, []) do
CurrentUser.without_safe_mode do RelatedTagCalculator.similar_tags_for_search(tag_query, current_user).take(max_tags).pluck(:name)
RelatedTagCalculator.similar_tags_for_search(tag_query).take(max_tags).pluck(:name)
end
end end
end end
end end

View File

@@ -47,7 +47,7 @@ class RelatedTagQuery
end end
def similar_tags def similar_tags
@similar_tags ||= RelatedTagCalculator.similar_tags_for_search(query, category: category_of).take(limit) @similar_tags ||= RelatedTagCalculator.similar_tags_for_search(query, user, category: category_of).take(limit)
end end
# Returns the top 20 most frequently added tags within the last 20 edits made by the user in the last hour. # Returns the top 20 most frequently added tags within the last 20 edits made by the user in the last hour.

View File

@@ -248,7 +248,7 @@ module Sources
end end
def related_posts(limit = 5) def related_posts(limit = 5)
CurrentUser.as_system { Post.tag_match(related_posts_search_query).paginate(1, limit: limit) } Post.system_tag_match(related_posts_search_query).paginate(1, limit: limit)
end end
memoize :related_posts memoize :related_posts

View File

@@ -68,10 +68,8 @@ class UploadService
def start! def start!
if Utils.is_downloadable?(source) if Utils.is_downloadable?(source)
CurrentUser.as_system do if Post.system_tag_match("source:#{canonical_source}").where.not(id: original_post_id).exists?
if Post.tag_match("source:#{canonical_source}").where.not(id: original_post_id).exists? raise ActiveRecord::RecordNotUnique.new("A post with source #{canonical_source} already exists")
raise ActiveRecord::RecordNotUnique.new("A post with source #{canonical_source} already exists")
end
end end
if Upload.where(source: source, status: "completed").exists? if Upload.where(source: source, status: "completed").exists?

View File

@@ -134,10 +134,8 @@ class Artist < ApplicationRecord
source = params.delete(:source) source = params.delete(:source)
if source.blank? && params[:name].present? if source.blank? && params[:name].present?
CurrentUser.without_safe_mode do post = Post.system_tag_match("source:http* #{params[:name]}").first
post = Post.tag_match("source:http* #{params[:name]}").first source = post.try(:source)
source = post.try(:source)
end
end end
if source.present? if source.present?
@@ -168,36 +166,32 @@ class Artist < ApplicationRecord
module BanMethods module BanMethods
def unban! def unban!
Post.transaction do Post.transaction do
CurrentUser.without_safe_mode do ti = TagImplication.find_by(antecedent_name: name, consequent_name: "banned_artist")
ti = TagImplication.find_by(antecedent_name: name, consequent_name: "banned_artist") ti&.destroy
ti&.destroy
Post.tag_match(name).find_each do |post| Post.raw_tag_match(name).find_each do |post|
post.unban! post.unban!
fixed_tags = post.tag_string.sub(/(?:\A| )banned_artist(?:\Z| )/, " ").strip fixed_tags = post.tag_string.sub(/(?:\A| )banned_artist(?:\Z| )/, " ").strip
post.update(tag_string: fixed_tags) post.update(tag_string: fixed_tags)
end
update!(is_banned: false)
ModAction.log("unbanned artist ##{id}", :artist_unban)
end end
update!(is_banned: false)
ModAction.log("unbanned artist ##{id}", :artist_unban)
end end
end end
def ban!(banner: CurrentUser.user) def ban!(banner: CurrentUser.user)
Post.transaction do Post.transaction do
CurrentUser.without_safe_mode do Post.raw_tag_match(name).each(&:ban!)
Post.tag_match(name).each(&:ban!)
# potential race condition but unlikely # potential race condition but unlikely
unless TagImplication.where(:antecedent_name => name, :consequent_name => "banned_artist").exists? unless TagImplication.where(:antecedent_name => name, :consequent_name => "banned_artist").exists?
tag_implication = TagImplication.create!(antecedent_name: name, consequent_name: "banned_artist", skip_secondary_validations: true, creator: banner) tag_implication = TagImplication.create!(antecedent_name: name, consequent_name: "banned_artist", skip_secondary_validations: true, creator: banner)
tag_implication.approve!(approver: banner) tag_implication.approve!(approver: banner)
end
update!(is_banned: true)
ModAction.log("banned artist ##{id}", :artist_ban)
end end
update!(is_banned: true)
ModAction.log("banned artist ##{id}", :artist_ban)
end end
end end
end end

View File

@@ -27,7 +27,7 @@ class Pool < ApplicationRecord
end end
def post_tags_match(query) def post_tags_match(query)
posts = Post.tag_match(query).select(:id).reorder(nil) posts = Post.user_tag_match(query).select(:id).reorder(nil)
pools = Pool.joins("CROSS JOIN unnest(post_ids) AS post_id").group(:id).where("post_id IN (?)", posts) pools = Pool.joins("CROSS JOIN unnest(post_ids) AS post_id").group(:id).where("post_id IN (?)", posts)
where(id: pools) where(id: pools)
end end

View File

@@ -604,7 +604,7 @@ class Post < ApplicationRecord
# If someone else committed changes to this post before we did, # If someone else committed changes to this post before we did,
# then try to merge the tag changes together. # then try to merge the tag changes together.
current_tags = tag_string_was.split current_tags = tag_string_was.split
new_tags = PostQueryBuilder.new(tag_string).parse_tag_edit new_tags = PostQueryBuilder.new(tag_string).split_query
old_tags = old_tag_string.split old_tags = old_tag_string.split
kept_tags = current_tags & new_tags kept_tags = current_tags & new_tags
@@ -642,7 +642,7 @@ class Post < ApplicationRecord
end end
def normalize_tags def normalize_tags
normalized_tags = PostQueryBuilder.new(tag_string).parse_tag_edit normalized_tags = PostQueryBuilder.new(tag_string).split_query
normalized_tags = apply_casesensitive_metatags(normalized_tags) normalized_tags = apply_casesensitive_metatags(normalized_tags)
normalized_tags = normalized_tags.map(&:downcase) normalized_tags = normalized_tags.map(&:downcase)
normalized_tags = filter_metatags(normalized_tags) normalized_tags = filter_metatags(normalized_tags)
@@ -1372,9 +1372,7 @@ class Post < ApplicationRecord
end end
def sample(query, sample_size) def sample(query, sample_size)
CurrentUser.without_safe_mode do user_tag_match(query, safe_mode: false, hide_deleted_posts: false).reorder(:md5).limit(sample_size)
tag_match(query).reorder(:md5).limit(sample_size)
end
end end
# unflattens the tag_string into one tag per row. # unflattens the tag_string into one tag per row.
@@ -1472,8 +1470,12 @@ class Post < ApplicationRecord
where("posts.tag_index @@ to_tsquery('danbooru', E?)", tag.to_escaped_for_tsquery) where("posts.tag_index @@ to_tsquery('danbooru', E?)", tag.to_escaped_for_tsquery)
end end
def tag_match(query) def system_tag_match(query)
PostQueryBuilder.new(query).build user_tag_match(query, User.system, safe_mode: false, hide_deleted_posts: false)
end
def user_tag_match(query, user = CurrentUser.user, safe_mode: CurrentUser.safe_mode?, hide_deleted_posts: user.hide_deleted_posts?)
PostQueryBuilder.new(query, user, safe_mode: safe_mode, hide_deleted_posts: hide_deleted_posts).build
end end
def search(params) def search(params)
@@ -1488,7 +1490,7 @@ class Post < ApplicationRecord
) )
if params[:tags].present? if params[:tags].present?
q = q.tag_match(params[:tags]) q = q.user_tag_match(params[:tags])
end end
if params[:order].present? if params[:order].present?

View File

@@ -114,18 +114,16 @@ class SavedSearch < ApplicationRecord
end end
def populate(query, timeout: 10_000) def populate(query, timeout: 10_000)
CurrentUser.as_system do redis_key = "search:#{query}"
redis_key = "search:#{query}" return if redis.exists(redis_key)
return if redis.exists(redis_key)
post_ids = Post.with_timeout(timeout, [], query: query) do post_ids = Post.with_timeout(timeout, [], query: query) do
Post.tag_match(query).limit(QUERY_LIMIT).pluck(:id) Post.system_tag_match(query).limit(QUERY_LIMIT).pluck(:id)
end end
if post_ids.present? if post_ids.present?
redis.sadd(redis_key, post_ids) redis.sadd(redis_key, post_ids)
redis.expire(redis_key, REDIS_EXPIRY.to_i) redis.expire(redis_key, REDIS_EXPIRY.to_i)
end
end end
end end
end end

View File

@@ -349,7 +349,7 @@ class Tag < ApplicationRecord
end end
def posts def posts
Post.tag_match(name) Post.system_tag_match(name)
end end
def self.available_includes def self.available_includes

View File

@@ -40,7 +40,7 @@ module PostSetPresenters
end end
def similar_tags def similar_tags
RelatedTagCalculator.cached_similar_tags_for_search(post_set.tag_string, MAX_TAGS) RelatedTagCalculator.cached_similar_tags_for_search(post_set.tag_string, MAX_TAGS, CurrentUser.user)
end end
def frequent_tags def frequent_tags

View File

@@ -36,11 +36,11 @@ class UserPresenter
end end
def posts_for_saved_search_category(category) def posts_for_saved_search_category(category)
Post.tag_match("search:#{category}").limit(10) Post.user_tag_match("search:#{category}").limit(10)
end end
def uploads def uploads
Post.tag_match("user:#{user.name}").limit(6) Post.user_tag_match("user:#{user.name}").limit(6)
end end
def has_uploads? def has_uploads?
@@ -48,7 +48,7 @@ class UserPresenter
end end
def favorites def favorites
Post.tag_match("ordfav:#{user.name}").limit(6) Post.user_tag_match("ordfav:#{user.name}").limit(6)
end end
def has_favorites? def has_favorites?
@@ -76,7 +76,7 @@ class UserPresenter
end end
def commented_posts_count(template) def commented_posts_count(template)
count = CurrentUser.without_safe_mode { PostQueryBuilder.new("commenter:#{user.name}").fast_count } count = PostQueryBuilder.new("commenter:#{user.name}", User.system).fast_count
count = "?" if count.nil? count = "?" if count.nil?
template.link_to(count, template.posts_path(:tags => "commenter:#{user.name} order:comment_bumped")) template.link_to(count, template.posts_path(:tags => "commenter:#{user.name} order:comment_bumped"))
end end
@@ -90,7 +90,7 @@ class UserPresenter
end end
def noted_posts_count(template) def noted_posts_count(template)
count = CurrentUser.without_safe_mode { PostQueryBuilder.new("noteupdater:#{user.name}").fast_count } count = PostQueryBuilder.new("noteupdater:#{user.name}", User.system).fast_count
count = "?" if count.nil? count = "?" if count.nil?
template.link_to(count, template.posts_path(:tags => "noteupdater:#{user.name} order:note")) template.link_to(count, template.posts_path(:tags => "noteupdater:#{user.name} order:note"))
end end

View File

@@ -2,11 +2,11 @@ require 'test_helper'
class PostQueryBuilderTest < ActiveSupport::TestCase class PostQueryBuilderTest < ActiveSupport::TestCase
def assert_tag_match(posts, query) def assert_tag_match(posts, query)
assert_equal(posts.map(&:id), Post.tag_match(query).pluck(:id)) assert_equal(posts.map(&:id), Post.user_tag_match(query).pluck(:id))
end end
def assert_fast_count(count, query, **options) def assert_fast_count(count, query, **options)
assert_equal(count, PostQueryBuilder.new(query).fast_count(**options)) assert_equal(count, PostQueryBuilder.new(query, **options).fast_count)
end end
setup do setup do
@@ -616,6 +616,7 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
assert_tag_match([deleted], "status:deleted") assert_tag_match([deleted], "status:deleted")
assert_tag_match([undeleted, deleted], "status:any") assert_tag_match([undeleted, deleted], "status:any")
assert_tag_match([undeleted, deleted], "status:all") assert_tag_match([undeleted, deleted], "status:all")
assert_tag_match([deleted], "status:banned status:deleted")
assert_tag_match([], "-status:banned") assert_tag_match([], "-status:banned")
assert_tag_match([deleted], "-status:active") assert_tag_match([deleted], "-status:active")
@@ -629,6 +630,8 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
assert_tag_match([deleted], "status:deleted") assert_tag_match([deleted], "status:deleted")
assert_tag_match([undeleted, deleted], "status:any") assert_tag_match([undeleted, deleted], "status:any")
assert_tag_match([undeleted, deleted], "status:all") assert_tag_match([undeleted, deleted], "status:all")
assert_fast_count(2, "status:banned")
end end
should "return posts for the filetype:<ext> metatag" do should "return posts for the filetype:<ext> metatag" do
@@ -988,7 +991,7 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
post1 = create(:post, rating: "s") post1 = create(:post, rating: "s")
assert_raise(::Post::SearchError) do assert_raise(::Post::SearchError) do
Post.tag_match("a b c rating:s width:10 height:10 user:bob") Post.user_tag_match("a b c rating:s width:10 height:10 user:bob")
end end
end end
@@ -1143,29 +1146,28 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
context "in safe mode" do context "in safe mode" do
setup do setup do
CurrentUser.stubs(:safe_mode?).returns(true)
create(:post, rating: "s") create(:post, rating: "s")
end end
should "work for a blank search" do should "work for a blank search" do
assert_fast_count(1, "") assert_fast_count(1, "", safe_mode: true)
end end
should "work for a nil search" do should "work for a nil search" do
assert_fast_count(1, nil) assert_fast_count(1, nil, safe_mode: true)
end end
should "not fail for a two tag search by a member" do should "not fail for a two tag search by a member" do
post1 = create(:post, tag_string: "aaa bbb rating:s") post1 = create(:post, tag_string: "aaa bbb rating:s")
post2 = create(:post, tag_string: "aaa bbb rating:e") post2 = create(:post, tag_string: "aaa bbb rating:e")
assert_fast_count(1, "aaa bbb") assert_fast_count(1, "aaa bbb", safe_mode: true)
end end
context "with a primed cache" do context "with a primed cache" do
should "fetch the value from the cache" do should "fetch the value from the cache" do
PostQueryBuilder.new(nil).set_count_in_cache("rating:s", 100) PostQueryBuilder.new(nil).set_count_in_cache("rating:s", 100)
assert_fast_count(100, "") assert_fast_count(100, "", safe_mode: true)
end end
end end
end end

View File

@@ -2,14 +2,7 @@ require 'test_helper'
class RelatedTagCalculatorTest < ActiveSupport::TestCase class RelatedTagCalculatorTest < ActiveSupport::TestCase
setup do setup do
user = FactoryBot.create(:user) @user = create(:user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
end
teardown do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end end
context "RelatedTagCalculator" do context "RelatedTagCalculator" do
@@ -18,7 +11,7 @@ class RelatedTagCalculatorTest < ActiveSupport::TestCase
create(:post, tag_string: "aaa bbb ccc ddd") create(:post, tag_string: "aaa bbb ccc ddd")
create(:post, tag_string: "aaa bbb ccc") create(:post, tag_string: "aaa bbb ccc")
create(:post, tag_string: "aaa bbb") create(:post, tag_string: "aaa bbb")
posts = Post.tag_match("aaa") posts = Post.user_tag_match("aaa", @user)
assert_equal(%w[aaa bbb ccc ddd], RelatedTagCalculator.frequent_tags_for_post_array(posts)) assert_equal(%w[aaa bbb ccc ddd], RelatedTagCalculator.frequent_tags_for_post_array(posts))
end end
@@ -30,7 +23,7 @@ class RelatedTagCalculatorTest < ActiveSupport::TestCase
create(:post, tag_string: "aaa bbb ccc") create(:post, tag_string: "aaa bbb ccc")
create(:post, tag_string: "aaa bbb") create(:post, tag_string: "aaa bbb")
assert_equal(%w[aaa bbb ccc ddd], RelatedTagCalculator.frequent_tags_for_search("aaa").pluck(:name)) assert_equal(%w[aaa bbb ccc ddd], RelatedTagCalculator.frequent_tags_for_search("aaa", @user).pluck(:name))
end end
should "calculate the most frequent tags for a multiple tag search" do should "calculate the most frequent tags for a multiple tag search" do
@@ -38,16 +31,16 @@ class RelatedTagCalculatorTest < ActiveSupport::TestCase
create(:post, tag_string: "aaa bbb ccc ddd") create(:post, tag_string: "aaa bbb ccc ddd")
create(:post, tag_string: "aaa eee fff") create(:post, tag_string: "aaa eee fff")
assert_equal(%w[aaa bbb ccc ddd], RelatedTagCalculator.frequent_tags_for_search("aaa bbb").pluck(:name)) assert_equal(%w[aaa bbb ccc ddd], RelatedTagCalculator.frequent_tags_for_search("aaa bbb", @user).pluck(:name))
end end
should "calculate the most frequent tags with a category constraint" do should "calculate the most frequent tags with a category constraint" do
create(:post, tag_string: "aaa bbb art:ccc copy:ddd") create(:post, tag_string: "aaa bbb art:ccc copy:ddd")
create(:post, tag_string: "aaa bbb art:ccc") create(:post, tag_string: "aaa bbb ccc")
create(:post, tag_string: "aaa bbb") create(:post, tag_string: "aaa bbb")
assert_equal(%w[aaa bbb], RelatedTagCalculator.frequent_tags_for_search("aaa", category: Tag.categories.general).pluck(:name)) assert_equal(%w[aaa bbb], RelatedTagCalculator.frequent_tags_for_search("aaa", @user, category: Tag.categories.general).pluck(:name))
assert_equal(%w[ccc], RelatedTagCalculator.frequent_tags_for_search("aaa", category: Tag.categories.artist).pluck(:name)) assert_equal(%w[ccc], RelatedTagCalculator.frequent_tags_for_search("aaa", @user, category: Tag.categories.artist).pluck(:name))
end end
end end
@@ -57,9 +50,9 @@ class RelatedTagCalculatorTest < ActiveSupport::TestCase
create(:post, tag_string: "1girl solo", rating: "q") create(:post, tag_string: "1girl solo", rating: "q")
create(:post, tag_string: "1girl 1boy", rating: "q") create(:post, tag_string: "1girl 1boy", rating: "q")
assert_equal(%w[1girl solo 1boy], RelatedTagCalculator.similar_tags_for_search("1girl").pluck(:name)) assert_equal(%w[1girl solo 1boy], RelatedTagCalculator.similar_tags_for_search("1girl", @user).pluck(:name))
assert_equal(%w[1girl 1boy solo], RelatedTagCalculator.similar_tags_for_search("rating:q").pluck(:name)) assert_equal(%w[1girl 1boy solo], RelatedTagCalculator.similar_tags_for_search("rating:q", @user).pluck(:name))
assert_equal(%w[solo 1girl], RelatedTagCalculator.similar_tags_for_search("solo").pluck(:name)) assert_equal(%w[solo 1girl], RelatedTagCalculator.similar_tags_for_search("solo", @user).pluck(:name))
end end
should "calculate the similar tags for an aliased tag" do should "calculate the similar tags for an aliased tag" do
@@ -67,7 +60,7 @@ class RelatedTagCalculatorTest < ActiveSupport::TestCase
create(:post, tag_string: "bunny dog") create(:post, tag_string: "bunny dog")
create(:post, tag_string: "bunny cat") create(:post, tag_string: "bunny cat")
assert_equal(%w[bunny cat dog], RelatedTagCalculator.similar_tags_for_search("rabbit").pluck(:name)) assert_equal(%w[bunny cat dog], RelatedTagCalculator.similar_tags_for_search("rabbit", @user).pluck(:name))
end end
end end
end end