jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

search: split tag_match into user_tag_match / system_tag_match.

Browse Source 
When doing a tag search, we have to be careful about which user we're
running the search as because the results depend on the current user.
Specifically, things like private favorites, private favorite groups,
post votes, saved searches, and flagger names depend on the user's
permissions, and whether non-safe or deleted posts are filtered out
depend on whether the user has safe mode on or the hide deleted posts
setting enabled.

* Refactor internal searches to explicitly state whether they're
  running as the system user (DanbooruBot) or as the current user.
* Explicitly pass in the current user to PostQueryBuilder instead of
  implicitly relying on the CurrentUser global.
* Get rid of CurrentUser.admin_mode? (used to ignore the hide deleted
  post setting) and CurrentUser.without_safe_mode (used to ignore safe
  mode).
* Change the /counts/posts.json endpoint to ignore safe mode and the
  hide deleted posts settings when counting posts.
* Fix searches not correctly overriding the hide deleted posts setting
  when multiple status: metatags were used (e.g. `status:banned status:active`)
* Fix fast_count not respecting the hide deleted posts setting when the
  status:banned metatag was used.
This commit is contained in:
evazion
2020-05-06 22:00:47 -05:00
parent a753ebbea9
commit f38c38f26e
24 changed files with 120 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/logical/concerns/searchable.rb
View File

@@ -226,7 +226,7 @@ module Searchable
end
if params[:post_tags_match].present?
relation = relation.where(post_id: Post.tag_match(params[:post_tags_match]).reorder(nil))
relation = relation.where(post_id: Post.user_tag_match(params[:post_tags_match]).reorder(nil))
end
relation

14
app/logical/current_user.rb
View File

@@ -73,20 +73,6 @@ class CurrentUser
RequestStore[:safe_mode]
end
def self.admin_mode?
RequestStore[:admin_mode]
end
def self.without_safe_mode
prev = RequestStore[:safe_mode]
RequestStore[:safe_mode] = false
RequestStore[:admin_mode] = true
yield
ensure
RequestStore[:safe_mode] = prev
RequestStore[:admin_mode] = false
end
def self.safe_mode=(safe_mode)
RequestStore[:safe_mode] = safe_mode
end

46
app/logical/post_query_builder.rb
View File

@@ -53,10 +53,15 @@ class PostQueryBuilder
COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } +
CATEGORY_COUNT_METATAGS.flat_map { |str| [str, "#{str}_asc"] }
attr_accessor :query_string
attr_reader :query_string, :current_user, :safe_mode, :hide_deleted_posts
alias_method :safe_mode?, :safe_mode
alias_method :hide_deleted_posts?, :hide_deleted_posts
def initialize(query_string)
def initialize(query_string, current_user = User.anonymous, safe_mode: false, hide_deleted_posts: false)
@query_string = query_string
@current_user = current_user
@safe_mode = safe_mode
@hide_deleted_posts = hide_deleted_posts
end
def tags_match(tags, relation)
@@ -177,9 +182,9 @@ class PostQueryBuilder
when "noteupdater"
user_subquery_matches(NoteVersion.unscoped, value, field: :updater)
when "upvoter", "upvote"
user_subquery_matches(PostVote.positive.visible(CurrentUser.user), value, field: :user)
user_subquery_matches(PostVote.positive.visible(current_user), value, field: :user)
when "downvoter", "downvote"
user_subquery_matches(PostVote.negative.visible(CurrentUser.user), value, field: :user)
user_subquery_matches(PostVote.negative.visible(current_user), value, field: :user)
when *CATEGORY_COUNT_METATAGS
short_category = name.delete_suffix("tags")
category = TagCategory.short_name_mapping[short_category]
@@ -248,16 +253,16 @@ class PostQueryBuilder
user_subquery_matches(flags, username) do |username|
flagger = User.find_by_name(username)
PostFlag.unscoped.creator_matches(flagger, CurrentUser.user)
PostFlag.unscoped.creator_matches(flagger, current_user)
end
end
def saved_search_matches(label)
case label.downcase
when "all"
Post.where(id: SavedSearch.post_ids_for(CurrentUser.id))
Post.where(id: SavedSearch.post_ids_for(current_user.id))
else
Post.where(id: SavedSearch.post_ids_for(CurrentUser.id, label: label))
Post.where(id: SavedSearch.post_ids_for(current_user.id, label: label))
end
end
@@ -287,8 +292,8 @@ class PostQueryBuilder
def disapproved_matches(query)
if query.downcase.in?(PostDisapproval::REASONS)
Post.where(disapprovals: PostDisapproval.where(reason: query.downcase))
elsif User.normalize_name(query) == CurrentUser.user.name
Post.where(disapprovals: PostDisapproval.where(user: CurrentUser.user))
elsif User.normalize_name(query) == current_user.name
Post.where(disapprovals: PostDisapproval.where(user: current_user))
else
Post.none
end
@@ -362,20 +367,20 @@ class PostQueryBuilder
def ordfavgroup_matches(query)
# XXX unify with FavoriteGroup#posts
favgroup = FavoriteGroup.visible(CurrentUser.user).name_or_id_matches(query, CurrentUser.user)
favgroup = FavoriteGroup.visible(current_user).name_or_id_matches(query, current_user)
favgroup_posts = favgroup.joins("CROSS JOIN unnest(favorite_groups.post_ids) WITH ORDINALITY AS row(post_id, favgroup_index)").select(:post_id, :favgroup_index)
Post.joins("JOIN (#{favgroup_posts.to_sql}) favgroup_posts ON favgroup_posts.post_id = posts.id").order("favgroup_posts.favgroup_index ASC")
end
def favgroup_matches(query)
favgroup = FavoriteGroup.visible(CurrentUser.user).name_or_id_matches(query, CurrentUser.user)
favgroup = FavoriteGroup.visible(current_user).name_or_id_matches(query, current_user)
Post.where(id: favgroup.select("unnest(post_ids)"))
end
def favorites_include(username)
favuser = User.find_by_name(username)
if favuser.present? && Pundit.policy!([CurrentUser.user, nil], favuser).can_see_favorites?
if favuser.present? && Pundit.policy!([current_user, nil], favuser).can_see_favorites?
tags_include("fav:#{favuser.id}")
else
Post.none
@@ -445,10 +450,9 @@ class PostQueryBuilder
relation
end
def hide_deleted_posts?
return false if CurrentUser.admin_mode?
return false if find_metatag(:status).to_s.downcase.in?(%w[deleted active any all])
return CurrentUser.user.hide_deleted_posts?
def hide_deleted?
has_status_metatag = select_metatags(:status).any? { |metatag| metatag.value.downcase.in?(%w[deleted active any all]) }
hide_deleted_posts? && !has_status_metatag
end
def build
@@ -458,8 +462,8 @@ class PostQueryBuilder
end
relation = Post.all
relation = relation.where(rating: 's') if CurrentUser.safe_mode?
relation = relation.undeleted if hide_deleted_posts?
relation = relation.where(rating: 's') if safe_mode?
relation = relation.undeleted if hide_deleted?
relation = add_joins(relation)
relation = metatags_match(metatags, relation)
relation = tags_match(tags, relation)
@@ -813,8 +817,8 @@ class PostQueryBuilder
concerning :CountMethods do
def fast_count(timeout: 1_000, raise_on_timeout: false, skip_cache: false)
tags = normalize_query(normalize_aliases: true)
tags += " rating:s" if CurrentUser.safe_mode?
tags += " -status:deleted" if CurrentUser.hide_deleted_posts? && !has_metatag?("status")
tags += " rating:s" if safe_mode?
tags += " -status:deleted" if hide_deleted?
tags = tags.strip
# Optimize some cases. these are just estimates but at these
@@ -852,7 +856,7 @@ class PostQueryBuilder
def fast_count_search(tags, timeout:, raise_on_timeout:)
count = Post.with_timeout(timeout, nil, tags: tags) do
Post.tag_match(tags).count
Post.user_tag_match(tags).count
end
if count.nil?

4
app/logical/post_search_context.rb
View File

@@ -10,9 +10,9 @@ class PostSearchContext
def post_id
if seq == "prev"
Post.tag_match(tags).where("posts.id > ?", id).reorder("posts.id asc").first.try(:id)
Post.user_tag_match(tags).where("posts.id > ?", id).reorder("posts.id asc").first.try(:id)
else
Post.tag_match(tags).where("posts.id < ?", id).reorder("posts.id desc").first.try(:id)
Post.user_tag_match(tags).where("posts.id < ?", id).reorder("posts.id desc").first.try(:id)
end
end

6
app/logical/post_sets/post.rb
View File

@@ -4,7 +4,7 @@ module PostSets
attr_reader :page, :random, :post_count, :format, :tag_string, :query
def initialize(tags, page = 1, per_page = nil, random: false, format: "html")
@query = PostQueryBuilder.new(tags)
@query = PostQueryBuilder.new(tags, CurrentUser.user)
@tag_string = tags
@page = page
@per_page = per_page
@@ -92,7 +92,7 @@ module PostSets
def get_random_posts
per_page.times.inject([]) do |all, x|
all << ::Post.tag_match(tag_string).random
all << ::Post.user_tag_match(tag_string).random
end.compact.uniq
end
@@ -103,7 +103,7 @@ module PostSets
if is_random?
temp = get_random_posts
else
temp = ::Post.tag_match(tag_string).where("true /* PostSets::Post#posts:2 */").paginate(page, :count => post_count, :limit => per_page)
temp = ::Post.user_tag_match(tag_string).where("true /* PostSets::Post#posts:2 */").paginate(page, :count => post_count, :limit => per_page)
end
end
end

2
app/logical/recommender_service.rb
View File

@@ -36,7 +36,7 @@ module RecommenderService
posts = posts.where.not(id: post.id) if post
posts = posts.where.not(uploader_id: uploader.id) if uploader
posts = posts.where.not(id: favoriter.favorites.select(:post_id)) if favoriter
posts = posts.where(id: Post.tag_match(tags).reorder(nil).select(:id)) if tags.present?
posts = posts.where(id: Post.user_tag_match(tags).reorder(nil).select(:id)) if tags.present?
id_to_score = recs.to_h
recs = posts.map { |post| { score: id_to_score[post.id], post: post } }

16
app/logical/related_tag_calculator.rb
View File

@@ -1,12 +1,12 @@
module RelatedTagCalculator
def self.similar_tags_for_search(tag_query, search_sample_size: 1000, tag_sample_size: 250, category: nil)
search_count = PostQueryBuilder.new(tag_query).fast_count
def self.similar_tags_for_search(tag_query, current_user, search_sample_size: 1000, tag_sample_size: 250, category: nil)
search_count = PostQueryBuilder.new(tag_query, current_user).fast_count
return [] if search_count.nil?
search_sample_size = [search_count, search_sample_size].min
return [] if search_sample_size <= 0
tags = frequent_tags_for_search(tag_query, search_sample_size: search_sample_size, category: category).limit(tag_sample_size)
tags = frequent_tags_for_search(tag_query, current_user, search_sample_size: search_sample_size, category: category).limit(tag_sample_size)
tags = tags.sort_by do |tag|
# cosine distance(tag1, tag2) = 1 - {{tag1 tag2}} / sqrt({{tag1}} * {{tag2}})
1 - tag.overlap_count / Math.sqrt(tag.post_count * search_count.to_f)
@@ -15,8 +15,8 @@ module RelatedTagCalculator
tags
end
def self.frequent_tags_for_search(tag_query, search_sample_size: 1000, category: nil)
sample_posts = Post.tag_match(tag_query).reorder(:md5).limit(search_sample_size)
def self.frequent_tags_for_search(tag_query, current_user, search_sample_size: 1000, category: nil)
sample_posts = Post.user_tag_match(tag_query, current_user).reorder(:md5).limit(search_sample_size)
frequent_tags_for_post_relation(sample_posts, category: category)
end
@@ -36,12 +36,10 @@ module RelatedTagCalculator
tags_with_counts.sort_by { |tag_name, count| [-count, tag_name] }.map(&:first)
end
def self.cached_similar_tags_for_search(tag_query, max_tags, search_timeout: 2000, cache_timeout: 8.hours)
def self.cached_similar_tags_for_search(tag_query, max_tags, current_user, search_timeout: 2000, cache_timeout: 8.hours)
Cache.get("similar_tags:#{tag_query}", cache_timeout, race_condition_ttl: 60.seconds) do
ApplicationRecord.with_timeout(search_timeout, []) do
CurrentUser.without_safe_mode do
RelatedTagCalculator.similar_tags_for_search(tag_query).take(max_tags).pluck(:name)
end
RelatedTagCalculator.similar_tags_for_search(tag_query, current_user).take(max_tags).pluck(:name)
end
end
end

2
app/logical/related_tag_query.rb
View File

@@ -47,7 +47,7 @@ class RelatedTagQuery
end
def similar_tags
@similar_tags ||= RelatedTagCalculator.similar_tags_for_search(query, category: category_of).take(limit)
@similar_tags ||= RelatedTagCalculator.similar_tags_for_search(query, user, category: category_of).take(limit)
end
# Returns the top 20 most frequently added tags within the last 20 edits made by the user in the last hour.

2
app/logical/sources/strategies/base.rb
View File

@@ -248,7 +248,7 @@ module Sources
end
def related_posts(limit = 5)
CurrentUser.as_system { Post.tag_match(related_posts_search_query).paginate(1, limit: limit) }
Post.system_tag_match(related_posts_search_query).paginate(1, limit: limit)
end
memoize :related_posts

6
app/logical/upload_service/preprocessor.rb
View File

@@ -68,10 +68,8 @@ class UploadService
def start!
if Utils.is_downloadable?(source)
CurrentUser.as_system do
if Post.tag_match("source:#{canonical_source}").where.not(id: original_post_id).exists?
raise ActiveRecord::RecordNotUnique.new("A post with source #{canonical_source} already exists")
end
if Post.system_tag_match("source:#{canonical_source}").where.not(id: original_post_id).exists?
raise ActiveRecord::RecordNotUnique.new("A post with source #{canonical_source} already exists")
end
if Upload.where(source: source, status: "completed").exists?