search: move query parsing code from tag model to post query builder.

This commit is contained in:
evazion
2020-03-06 21:14:50 -06:00
parent 2e0ad42eca
commit 967d398c8e
13 changed files with 583 additions and 598 deletions

View File

@@ -25,7 +25,7 @@ module PostsHelper
return unless post_search_counts_enabled? return unless post_search_counts_enabled?
return unless params[:action] == "index" && params[:page].nil? && params[:tags].present? return unless params[:action] == "index" && params[:page].nil? && params[:tags].present?
tags = Tag.scan_query(params[:tags]).sort.join(" ") tags = PostQueryBuilder.scan_query(params[:tags]).sort.join(" ")
sig = generate_reportbooru_signature("ps-#{tags}") sig = generate_reportbooru_signature("ps-#{tags}")
render "posts/partials/index/search_count", sig: sig render "posts/partials/index/search_count", sig: sig
end end
@@ -63,7 +63,7 @@ module PostsHelper
end end
def show_tag_change_notice? def show_tag_change_notice?
Tag.scan_query(params[:tags]).size == 1 && TagChangeNoticeService.get_forum_topic_id(params[:tags]) PostQueryBuilder.scan_query(params[:tags]).size == 1 && TagChangeNoticeService.get_forum_topic_id(params[:tags])
end end
private private

View File

@@ -3,9 +3,9 @@ import CurrentUser from './current_user'
let Autocomplete = {}; let Autocomplete = {};
/* eslint-disable */ /* eslint-disable */
Autocomplete.METATAGS = <%= Tag::METATAGS.to_json.html_safe %>; Autocomplete.METATAGS = <%= PostQueryBuilder::METATAGS.to_json.html_safe %>;
Autocomplete.TAG_CATEGORIES = <%= TagCategory.mapping.to_json.html_safe %>; Autocomplete.TAG_CATEGORIES = <%= TagCategory.mapping.to_json.html_safe %>;
Autocomplete.ORDER_METATAGS = <%= Tag::ORDER_METATAGS.to_json.html_safe %>; Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_safe %>;
/* eslint-enable */ /* eslint-enable */
Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|"); Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|");

View File

@@ -6,8 +6,8 @@ class TagBatchChangeJob < ApplicationJob
def perform(antecedent, consequent, updater, updater_ip_addr) def perform(antecedent, consequent, updater, updater_ip_addr)
raise Error.new("antecedent is missing") if antecedent.blank? raise Error.new("antecedent is missing") if antecedent.blank?
normalized_antecedent = TagAlias.to_aliased(::Tag.scan_tags(antecedent.mb_chars.downcase)) normalized_antecedent = TagAlias.to_aliased(PostQueryBuilder.scan_query(antecedent.mb_chars.downcase))
normalized_consequent = TagAlias.to_aliased(::Tag.scan_tags(consequent.mb_chars.downcase)) normalized_consequent = TagAlias.to_aliased(PostQueryBuilder.scan_query(consequent.mb_chars.downcase))
CurrentUser.without_safe_mode do CurrentUser.without_safe_mode do
CurrentUser.scoped(updater, updater_ip_addr) do CurrentUser.scoped(updater, updater_ip_addr) do
@@ -30,7 +30,7 @@ class TagBatchChangeJob < ApplicationJob
end end
def migrate_saved_searches(normalized_antecedent, normalized_consequent) def migrate_saved_searches(normalized_antecedent, normalized_consequent)
tags = Tag.scan_tags(normalized_antecedent.join(" "), strip_metatags: true) tags = PostQueryBuilder.scan_query(normalized_antecedent.join(" "), strip_metatags: true)
# https://www.postgresql.org/docs/current/static/functions-array.html # https://www.postgresql.org/docs/current/static/functions-array.html
saved_searches = SavedSearch.where("string_to_array(query, ' ') @> ARRAY[?]", tags) saved_searches = SavedSearch.where("string_to_array(query, ' ') @> ARRAY[?]", tags)
@@ -53,7 +53,7 @@ class TagBatchChangeJob < ApplicationJob
begin begin
repl = user.blacklisted_tags.split(/\r\n|\r|\n/).map do |line| repl = user.blacklisted_tags.split(/\r\n|\r|\n/).map do |line|
list = Tag.scan_tags(line) list = PostQueryBuilder.scan_query(line)
if (list & query).size != query.size if (list & query).size != query.size
next line next line

View File

@@ -88,8 +88,8 @@ class AliasAndImplicationImporter
all all
when :mass_update when :mass_update
all += Tag.scan_tags(token[1]) all += PostQueryBuilder.scan_query(token[1])
all += Tag.scan_tags(token[2]) all += PostQueryBuilder.scan_query(token[2])
all all
when :change_category when :change_category

View File

@@ -64,7 +64,7 @@ module Searchable
def where_array_count(attr, value) def where_array_count(attr, value)
relation = all relation = all
qualified_column = "cardinality(#{qualified_column_for(attr)})" qualified_column = "cardinality(#{qualified_column_for(attr)})"
parsed_range = Tag.parse_helper(value, :integer) parsed_range = PostQueryBuilder.parse_helper(value, :integer)
PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, relation) PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, relation)
end end
@@ -96,7 +96,7 @@ module Searchable
column = column_for_attribute(attribute) column = column_for_attribute(attribute)
qualified_column = "#{table_name}.#{column.name}" qualified_column = "#{table_name}.#{column.name}"
parsed_range = Tag.parse_helper(range, column.type) parsed_range = PostQueryBuilder.parse_helper(range, column.type)
PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, self) PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, self)
end end
@@ -252,7 +252,7 @@ module Searchable
def apply_default_order(params) def apply_default_order(params)
if params[:order] == "custom" if params[:order] == "custom"
parse_ids = Tag.parse_helper(params[:id]) parse_ids = PostQueryBuilder.parse_helper(params[:id])
if parse_ids[0] == :in if parse_ids[0] == :in
return find_ordered(parse_ids[1]) return find_ordered(parse_ids[1])
end end

View File

@@ -1,4 +1,49 @@
class PostQueryBuilder class PostQueryBuilder
COUNT_METATAGS = %w[
comment_count deleted_comment_count active_comment_count
note_count deleted_note_count active_note_count
flag_count resolved_flag_count unresolved_flag_count
child_count deleted_child_count active_child_count
pool_count deleted_pool_count active_pool_count series_pool_count collection_pool_count
appeal_count approval_count replacement_count
]
# allow e.g. `deleted_comments` as a synonym for `deleted_comment_count`
COUNT_METATAG_SYNONYMS = COUNT_METATAGS.map { |str| str.delete_suffix("_count").pluralize }
METATAGS = %w[
-user user -approver approver commenter comm noter noteupdater artcomm
-pool pool ordpool -favgroup favgroup -fav fav ordfav md5 -rating rating
-locked locked width height mpixels ratio score favcount filesize source
-source id -id date age order limit -status status tagcount parent -parent
child pixiv_id pixiv search upvote downvote filetype -filetype flagger
-flagger appealer -appealer disapproved -disapproved embedded
] + TagCategory.short_name_list.map {|x| "#{x}tags"} + COUNT_METATAGS + COUNT_METATAG_SYNONYMS
ORDER_METATAGS = %w[
id id_desc
score score_asc
favcount favcount_asc
created_at created_at_asc
change change_asc
comment comment_asc
comment_bumped comment_bumped_asc
note note_asc
artcomm artcomm_asc
mpixels mpixels_asc
portrait landscape
filesize filesize_asc
tagcount tagcount_asc
rank
curated
modqueue
random
custom
] +
COUNT_METATAGS +
COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } +
TagCategory.short_name_list.flat_map { |str| ["#{str}tags", "#{str}tags_asc"] }
attr_accessor :query_string attr_accessor :query_string
def initialize(query_string) def initialize(query_string)
@@ -81,7 +126,7 @@ class PostQueryBuilder
end end
def table_for_metatag(metatag) def table_for_metatag(metatag)
if metatag.in?(Tag::COUNT_METATAGS) if metatag.in?(COUNT_METATAGS)
metatag[/(?<table>[a-z]+)_count\z/i, :table] metatag[/(?<table>[a-z]+)_count\z/i, :table]
else else
nil nil
@@ -111,7 +156,7 @@ class PostQueryBuilder
def build def build
unless query_string.is_a?(Hash) unless query_string.is_a?(Hash)
q = Tag.parse_query(query_string) q = PostQueryBuilder.parse_query(query_string)
end end
relation = Post.all relation = Post.all
@@ -140,7 +185,7 @@ class PostQueryBuilder
end end
relation = add_range_relation(q[:post_tag_count], "posts.tag_count", relation) relation = add_range_relation(q[:post_tag_count], "posts.tag_count", relation)
Tag::COUNT_METATAGS.each do |column| COUNT_METATAGS.each do |column|
relation = add_range_relation(q[column.to_sym], "posts.#{column}", relation) relation = add_range_relation(q[column.to_sym], "posts.#{column}", relation)
end end
@@ -582,7 +627,7 @@ class PostQueryBuilder
when "filesize_asc" when "filesize_asc"
relation = relation.order("posts.file_size ASC") relation = relation.order("posts.file_size ASC")
when /\A(?<column>#{Tag::COUNT_METATAGS.join("|")})(_(?<direction>asc|desc))?\z/i when /\A(?<column>#{COUNT_METATAGS.join("|")})(_(?<direction>asc|desc))?\z/i
column = $~[:column] column = $~[:column]
direction = $~[:direction] || "desc" direction = $~[:direction] || "desc"
relation = relation.order(column => direction, :id => direction) relation = relation.order(column => direction, :id => direction)
@@ -624,4 +669,492 @@ class PostQueryBuilder
relation relation
end end
concerning :ParseMethods do
class_methods do
def scan_query(query, strip_metatags: false)
tagstr = query.to_s.gsub(/\u3000/, " ").strip
list = tagstr.scan(/-?source:".*?"/) || []
list += tagstr.gsub(/-?source:".*?"/, "").scan(/[^[:space:]]+/).uniq
list = list.map { |tag| tag.sub(/^[-~]/, "") } if strip_metatags
list
end
def normalize_query(query, normalize_aliases: true, sort: true)
tags = scan_query(query.to_s)
tags = tags.map { |t| Tag.normalize_name(t) }
tags = TagAlias.to_aliased(tags) if normalize_aliases
tags = tags.sort if sort
tags = tags.uniq
tags.join(" ")
end
def parse_query(query, options = {})
q = {}
q[:tag_count] = 0
q[:tags] = {
:related => [],
:include => [],
:exclude => []
}
scan_query(query).each do |token|
q[:tag_count] += 1 unless Danbooru.config.is_unlimited_tag?(token)
if token =~ /\A(#{METATAGS.join("|")}):(.+)\z/i
g1 = $1.downcase
g2 = $2
case g1
when "-user"
q[:uploader_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:uploader_id_neg] << user_id unless user_id.blank?
when "user"
user_id = User.name_to_id(g2)
q[:uploader_id] = user_id unless user_id.blank?
when "-approver"
if g2 == "none"
q[:approver_id] = "any"
elsif g2 == "any"
q[:approver_id] = "none"
else
q[:approver_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:approver_id_neg] << user_id unless user_id.blank?
end
when "approver"
if g2 == "none"
q[:approver_id] = "none"
elsif g2 == "any"
q[:approver_id] = "any"
else
user_id = User.name_to_id(g2)
q[:approver_id] = user_id unless user_id.blank?
end
when "flagger"
q[:flagger_ids] ||= []
if g2 == "none"
q[:flagger_ids] << "none"
elsif g2 == "any"
q[:flagger_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:flagger_ids] << user_id unless user_id.blank?
end
when "-flagger"
if g2 == "none"
q[:flagger_ids] ||= []
q[:flagger_ids] << "any"
elsif g2 == "any"
q[:flagger_ids] ||= []
q[:flagger_ids] << "none"
else
q[:flagger_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:flagger_ids_neg] << user_id unless user_id.blank?
end
when "appealer"
q[:appealer_ids] ||= []
if g2 == "none"
q[:appealer_ids] << "none"
elsif g2 == "any"
q[:appealer_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:appealer_ids] << user_id unless user_id.blank?
end
when "-appealer"
if g2 == "none"
q[:appealer_ids] ||= []
q[:appealer_ids] << "any"
elsif g2 == "any"
q[:appealer_ids] ||= []
q[:appealer_ids] << "none"
else
q[:appealer_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:appealer_ids_neg] << user_id unless user_id.blank?
end
when "commenter", "comm"
q[:commenter_ids] ||= []
if g2 == "none"
q[:commenter_ids] << "none"
elsif g2 == "any"
q[:commenter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:commenter_ids] << user_id unless user_id.blank?
end
when "noter"
q[:noter_ids] ||= []
if g2 == "none"
q[:noter_ids] << "none"
elsif g2 == "any"
q[:noter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:noter_ids] << user_id unless user_id.blank?
end
when "noteupdater"
q[:note_updater_ids] ||= []
user_id = User.name_to_id(g2)
q[:note_updater_ids] << user_id unless user_id.blank?
when "artcomm"
q[:artcomm_ids] ||= []
user_id = User.name_to_id(g2)
q[:artcomm_ids] << user_id unless user_id.blank?
when "disapproved"
q[:disapproved] ||= []
q[:disapproved] << g2
when "-disapproved"
q[:disapproved_neg] ||= []
q[:disapproved_neg] << g2
when "-pool"
q[:pool_neg] ||= []
q[:pool_neg] << g2
when "pool"
q[:pool] ||= []
q[:pool] << g2
when "ordpool"
q[:ordpool] = g2
when "-favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups_neg] ||= []
q[:favgroups_neg] << favgroup
when "favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups] ||= []
q[:favgroups] << favgroup
when "-fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:exclude] << "fav:#{User.name_to_id(g2)}"
when "fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{User.name_to_id(g2)}"
when "ordfav"
user_id = User.name_to_id(g2)
favuser = User.find(user_id)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{user_id}"
q[:ordfav] = user_id
when "search"
q[:saved_searches] ||= []
q[:saved_searches] << g2
when "md5"
q[:md5] = g2.downcase.split(/,/)
when "-rating"
q[:rating_negated] = g2.downcase
when "rating"
q[:rating] = g2.downcase
when "-locked"
q[:locked_negated] = g2.downcase
when "locked"
q[:locked] = g2.downcase
when "id"
q[:post_id] = parse_helper(g2)
when "-id"
q[:post_id_negated] = g2.to_i
when "width"
q[:width] = parse_helper(g2)
when "height"
q[:height] = parse_helper(g2)
when "mpixels"
q[:mpixels] = parse_helper_fudged(g2, :float)
when "ratio"
q[:ratio] = parse_helper(g2, :ratio)
when "score"
q[:score] = parse_helper(g2)
when "favcount"
q[:fav_count] = parse_helper(g2)
when "filesize"
q[:filesize] = parse_helper_fudged(g2, :filesize)
when "source"
q[:source] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "-source"
q[:source_neg] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "date"
q[:date] = parse_helper(g2, :date)
when "age"
q[:age] = reverse_parse_helper(parse_helper(g2, :age))
when "tagcount"
q[:post_tag_count] = parse_helper(g2)
when /(#{TagCategory.short_name_regex})tags/
q["#{TagCategory.short_name_mapping[$1]}_tag_count".to_sym] = parse_helper(g2)
when "parent"
q[:parent] = g2.downcase
when "-parent"
if g2.downcase == "none"
q[:parent] = "any"
elsif g2.downcase == "any"
q[:parent] = "none"
else
q[:parent_neg_ids] ||= []
q[:parent_neg_ids] << g2.downcase
end
when "child"
q[:child] = g2.downcase
when "order"
g2 = g2.downcase
order, suffix, _tail = g2.partition(/_(asc|desc)\z/i)
if order.in?(COUNT_METATAG_SYNONYMS)
g2 = order.singularize + "_count" + suffix
end
q[:order] = g2
when "limit"
# Do nothing. The controller takes care of it.
when "-status"
q[:status_neg] = g2.downcase
when "status"
q[:status] = g2.downcase
when "embedded"
q[:embedded] = g2.downcase
when "filetype"
q[:filetype] = g2.downcase
when "-filetype"
q[:filetype_neg] = g2.downcase
when "pixiv_id", "pixiv"
if g2.downcase == "any" || g2.downcase == "none"
q[:pixiv_id] = g2.downcase
else
q[:pixiv_id] = parse_helper(g2)
end
when "upvote"
if CurrentUser.user.is_admin?
q[:upvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:upvote] = CurrentUser.user
end
when "downvote"
if CurrentUser.user.is_admin?
q[:downvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:downvote] = CurrentUser.user
end
when *COUNT_METATAGS
q[g1.to_sym] = parse_helper(g2)
when *COUNT_METATAG_SYNONYMS
g1 = "#{g1.singularize}_count"
q[g1.to_sym] = parse_helper(g2)
end
else
parse_tag(token, q[:tags])
end
end
q[:tags][:exclude] = TagAlias.to_aliased(q[:tags][:exclude])
q[:tags][:include] = TagAlias.to_aliased(q[:tags][:include])
q[:tags][:related] = TagAlias.to_aliased(q[:tags][:related])
return q
end
def parse_tag(tag, output)
if tag[0] == "-" && tag.size > 1
output[:exclude] << tag[1..-1].mb_chars.downcase
elsif tag[0] == "~" && tag.size > 1
output[:include] << tag[1..-1].mb_chars.downcase
elsif tag =~ /\*/
matches = Tag.name_matches(tag).select("name").limit(Danbooru.config.tag_query_limit).order("post_count DESC").map(&:name)
matches = ["~no_matches~"] if matches.empty?
output[:include] += matches
else
output[:related] << tag.mb_chars.downcase
end
end
def parse_cast(object, type)
case type
when :integer
object.to_i
when :float
object.to_f
when :date, :datetime
Time.zone.parse(object) rescue nil
when :age
DurationParser.parse(object).ago
when :ratio
object =~ /\A(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)\Z/i
if $1 && $2.to_f != 0.0
($1.to_f / $2.to_f).round(2)
else
object.to_f.round(2)
end
when :filesize
object =~ /\A(\d+(?:\.\d*)?|\d*\.\d+)([kKmM]?)[bB]?\Z/
size = $1.to_f
unit = $2
conversion_factor = case unit
when /m/i
1024 * 1024
when /k/i
1024
else
1
end
(size * conversion_factor).to_i
end
end
def parse_helper(range, type = :integer)
# "1", "0.5", "5.", ".5":
# (-?(\d+(\.\d*)?|\d*\.\d+))
case range
when /\A(.+?)\.\.(.+)/
return [:between, parse_cast($1, type), parse_cast($2, type)]
when /\A<=(.+)/, /\A\.\.(.+)/
return [:lte, parse_cast($1, type)]
when /\A<(.+)/
return [:lt, parse_cast($1, type)]
when /\A>=(.+)/, /\A(.+)\.\.\Z/
return [:gte, parse_cast($1, type)]
when /\A>(.+)/
return [:gt, parse_cast($1, type)]
when /[, ]/
return [:in, range.split(/[, ]+/).map {|x| parse_cast(x, type)}]
else
return [:eq, parse_cast(range, type)]
end
end
def parse_helper_fudged(range, type)
result = parse_helper(range, type)
# Don't fudge the filesize when searching filesize:123b or filesize:123.
if result[0] == :eq && type == :filesize && range !~ /[km]b?\Z/i
result
elsif result[0] == :eq
new_min = (result[1] * 0.95).to_i
new_max = (result[1] * 1.05).to_i
[:between, new_min, new_max]
else
result
end
end
def reverse_parse_helper(array)
case array[0]
when :between
[:between, *array[1..-1].reverse]
when :lte
[:gte, *array[1..-1]]
when :lt
[:gt, *array[1..-1]]
when :gte
[:lte, *array[1..-1]]
when :gt
[:lt, *array[1..-1]]
else
array
end
end
end
end
end end

View File

@@ -4,7 +4,7 @@ module PostSets
attr_reader :tag_array, :page, :raw, :random, :post_count, :format attr_reader :tag_array, :page, :raw, :random, :post_count, :format
def initialize(tags, page = 1, per_page = nil, raw: false, random: false, format: "html") def initialize(tags, page = 1, per_page = nil, raw: false, random: false, format: "html")
@tag_array = Tag.scan_query(tags) @tag_array = PostQueryBuilder.scan_query(tags)
@page = page @page = page
@per_page = per_page @per_page = per_page
@raw = raw.to_s.truthy? @raw = raw.to_s.truthy?

View File

@@ -21,7 +21,7 @@ class TagNameValidator < ActiveModel::EachValidator
record.errors[attribute] << "'#{value}' cannot contain non-printable characters" record.errors[attribute] << "'#{value}' cannot contain non-printable characters"
when /[^[:ascii:]]/ when /[^[:ascii:]]/
record.errors[attribute] << "'#{value}' must consist of only ASCII characters" record.errors[attribute] << "'#{value}' must consist of only ASCII characters"
when /\A(#{Tag::METATAGS.join("|")}):(.+)\z/i when /\A(#{PostQueryBuilder::METATAGS.join("|")}):(.+)\z/i
record.errors[attribute] << "'#{value}' cannot begin with '#{$1}:'" record.errors[attribute] << "'#{value}' cannot begin with '#{$1}:'"
when /\A(#{Tag.categories.regexp}):(.+)\z/i when /\A(#{Tag.categories.regexp}):(.+)\z/i
record.errors[attribute] << "'#{value}' cannot begin with '#{$1}:'" record.errors[attribute] << "'#{value}' cannot begin with '#{$1}:'"

View File

@@ -522,11 +522,11 @@ class Post < ApplicationRecord
module TagMethods module TagMethods
def tag_array def tag_array
@tag_array ||= Tag.scan_tags(tag_string) @tag_array ||= PostQueryBuilder.scan_query(tag_string)
end end
def tag_array_was def tag_array_was
@tag_array_was ||= Tag.scan_tags(tag_string_in_database.presence || tag_string_before_last_save || "") @tag_array_was ||= PostQueryBuilder.scan_query(tag_string_in_database.presence || tag_string_before_last_save || "")
end end
def tags def tags
@@ -590,7 +590,7 @@ class Post < ApplicationRecord
# then try to merge the tag changes together. # then try to merge the tag changes together.
current_tags = tag_array_was current_tags = tag_array_was
new_tags = tag_array new_tags = tag_array
old_tags = Tag.scan_tags(old_tag_string) old_tags = PostQueryBuilder.scan_query(old_tag_string)
kept_tags = current_tags & new_tags kept_tags = current_tags & new_tags
@removed_tags = old_tags - kept_tags @removed_tags = old_tags - kept_tags
@@ -627,7 +627,7 @@ class Post < ApplicationRecord
end end
def normalize_tags def normalize_tags
normalized_tags = Tag.scan_tags(tag_string) normalized_tags = PostQueryBuilder.scan_query(tag_string)
normalized_tags = apply_casesensitive_metatags(normalized_tags) normalized_tags = apply_casesensitive_metatags(normalized_tags)
normalized_tags = normalized_tags.map(&:downcase) normalized_tags = normalized_tags.map(&:downcase)
normalized_tags = filter_metatags(normalized_tags) normalized_tags = filter_metatags(normalized_tags)
@@ -1058,7 +1058,7 @@ class Post < ApplicationRecord
tags = tags.to_s tags = tags.to_s
tags += " rating:s" if CurrentUser.safe_mode? tags += " rating:s" if CurrentUser.safe_mode?
tags += " -status:deleted" if CurrentUser.hide_deleted_posts? && !Tag.has_metatag?(tags, "status", "-status") tags += " -status:deleted" if CurrentUser.hide_deleted_posts? && !Tag.has_metatag?(tags, "status", "-status")
tags = Tag.normalize_query(tags) tags = PostQueryBuilder.normalize_query(tags)
# Optimize some cases. these are just estimates but at these # Optimize some cases. these are just estimates but at these
# quantities being off by a few hundred doesn't matter much # quantities being off by a few hundred doesn't matter much

View File

@@ -139,18 +139,18 @@ class SavedSearch < ApplicationRecord
.where(user_id: user_id) .where(user_id: user_id)
.labeled(label) .labeled(label)
.pluck(:query) .pluck(:query)
.map {|x| Tag.normalize_query(x, sort: true)} .map {|x| PostQueryBuilder.normalize_query(x, sort: true)}
.sort .sort
.uniq .uniq
end end
end end
def normalized_query def normalized_query
Tag.normalize_query(query, sort: true) PostQueryBuilder.normalize_query(query, sort: true)
end end
def normalize_query def normalize_query
self.query = Tag.normalize_query(query, sort: false) self.query = PostQueryBuilder.normalize_query(query, sort: false)
end end
end end

View File

@@ -1,51 +1,4 @@
class Tag < ApplicationRecord class Tag < ApplicationRecord
COUNT_METATAGS = %w[
comment_count deleted_comment_count active_comment_count
note_count deleted_note_count active_note_count
flag_count resolved_flag_count unresolved_flag_count
child_count deleted_child_count active_child_count
pool_count deleted_pool_count active_pool_count series_pool_count collection_pool_count
appeal_count approval_count replacement_count
]
# allow e.g. `deleted_comments` as a synonym for `deleted_comment_count`
COUNT_METATAG_SYNONYMS = COUNT_METATAGS.map { |str| str.delete_suffix("_count").pluralize }
METATAGS = %w[
-user user -approver approver commenter comm noter noteupdater artcomm
-pool pool ordpool -favgroup favgroup -fav fav ordfav md5 -rating rating
-locked locked width height mpixels ratio score favcount filesize source
-source id -id date age order limit -status status tagcount parent -parent
child pixiv_id pixiv search upvote downvote filetype -filetype flagger
-flagger appealer -appealer disapproved -disapproved embedded
] + TagCategory.short_name_list.map {|x| "#{x}tags"} + COUNT_METATAGS + COUNT_METATAG_SYNONYMS
SUBQUERY_METATAGS = %w[commenter comm noter noteupdater artcomm flagger -flagger appealer -appealer]
ORDER_METATAGS = %w[
id id_desc
score score_asc
favcount favcount_asc
created_at created_at_asc
change change_asc
comment comment_asc
comment_bumped comment_bumped_asc
note note_asc
artcomm artcomm_asc
mpixels mpixels_asc
portrait landscape
filesize filesize_asc
tagcount tagcount_asc
rank
curated
modqueue
random
custom
] +
COUNT_METATAGS +
COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } +
TagCategory.short_name_list.flat_map { |str| ["#{str}tags", "#{str}tags_asc"] }
has_one :wiki_page, :foreign_key => "title", :primary_key => "name" has_one :wiki_page, :foreign_key => "title", :primary_key => "name"
has_one :artist, :foreign_key => "name", :primary_key => "name" has_one :artist, :foreign_key => "name", :primary_key => "name"
has_one :antecedent_alias, -> {active}, :class_name => "TagAlias", :foreign_key => "antecedent_name", :primary_key => "name" has_one :antecedent_alias, -> {active}, :class_name => "TagAlias", :foreign_key => "antecedent_name", :primary_key => "name"
@@ -268,169 +221,17 @@ class Tag < ApplicationRecord
end end
module ParseMethods module ParseMethods
def normalize(query)
query.to_s.gsub(/\u3000/, " ").strip
end
def normalize_query(query, normalize_aliases: true, sort: true)
tags = Tag.scan_query(query.to_s)
tags = tags.map { |t| Tag.normalize_name(t) }
tags = TagAlias.to_aliased(tags) if normalize_aliases
tags = tags.sort if sort
tags = tags.uniq
tags.join(" ")
end
def scan_query(query)
tagstr = normalize(query)
list = tagstr.scan(/-?source:".*?"/) || []
list + tagstr.gsub(/-?source:".*?"/, "").scan(/[^[:space:]]+/).uniq
end
def scan_tags(tags, options = {})
tagstr = normalize(tags)
list = tagstr.scan(/source:".*?"/) || []
list += tagstr.gsub(/source:".*?"/, "").scan(/[^[:space:]]+/).uniq
if options[:strip_metatags]
list = list.map {|x| x.sub(/^[-~]/, "")}
end
list
end
def parse_cast(object, type)
case type
when :integer
object.to_i
when :float
object.to_f
when :date, :datetime
Time.zone.parse(object) rescue nil
when :age
DurationParser.parse(object).ago
when :ratio
object =~ /\A(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)\Z/i
if $1 && $2.to_f != 0.0
($1.to_f / $2.to_f).round(2)
else
object.to_f.round(2)
end
when :filesize
object =~ /\A(\d+(?:\.\d*)?|\d*\.\d+)([kKmM]?)[bB]?\Z/
size = $1.to_f
unit = $2
conversion_factor = case unit
when /m/i
1024 * 1024
when /k/i
1024
else
1
end
(size * conversion_factor).to_i
end
end
def parse_helper(range, type = :integer)
# "1", "0.5", "5.", ".5":
# (-?(\d+(\.\d*)?|\d*\.\d+))
case range
when /\A(.+?)\.\.(.+)/
return [:between, parse_cast($1, type), parse_cast($2, type)]
when /\A<=(.+)/, /\A\.\.(.+)/
return [:lte, parse_cast($1, type)]
when /\A<(.+)/
return [:lt, parse_cast($1, type)]
when /\A>=(.+)/, /\A(.+)\.\.\Z/
return [:gte, parse_cast($1, type)]
when /\A>(.+)/
return [:gt, parse_cast($1, type)]
when /[, ]/
return [:in, range.split(/[, ]+/).map {|x| parse_cast(x, type)}]
else
return [:eq, parse_cast(range, type)]
end
end
def parse_helper_fudged(range, type)
result = parse_helper(range, type)
# Don't fudge the filesize when searching filesize:123b or filesize:123.
if result[0] == :eq && type == :filesize && range !~ /[km]b?\Z/i
result
elsif result[0] == :eq
new_min = (result[1] * 0.95).to_i
new_max = (result[1] * 1.05).to_i
[:between, new_min, new_max]
else
result
end
end
def reverse_parse_helper(array)
case array[0]
when :between
[:between, *array[1..-1].reverse]
when :lte
[:gte, *array[1..-1]]
when :lt
[:gt, *array[1..-1]]
when :gte
[:lte, *array[1..-1]]
when :gt
[:lt, *array[1..-1]]
else
array
end
end
def parse_tag(tag, output)
if tag[0] == "-" && tag.size > 1
output[:exclude] << tag[1..-1].mb_chars.downcase
elsif tag[0] == "~" && tag.size > 1
output[:include] << tag[1..-1].mb_chars.downcase
elsif tag =~ /\*/
matches = Tag.name_matches(tag).select("name").limit(Danbooru.config.tag_query_limit).order("post_count DESC").map(&:name)
matches = ["~no_matches~"] if matches.empty?
output[:include] += matches
else
output[:related] << tag.mb_chars.downcase
end
end
# true if query is a single "simple" tag (not a metatag, negated tag, or wildcard tag). # true if query is a single "simple" tag (not a metatag, negated tag, or wildcard tag).
def is_simple_tag?(query) def is_simple_tag?(query)
is_single_tag?(query) && !is_metatag?(query) && !is_negated_tag?(query) && !is_optional_tag?(query) && !is_wildcard_tag?(query) is_single_tag?(query) && !is_metatag?(query) && !is_negated_tag?(query) && !is_optional_tag?(query) && !is_wildcard_tag?(query)
end end
def is_single_tag?(query) def is_single_tag?(query)
scan_query(query).size == 1 PostQueryBuilder.scan_query(query).size == 1
end end
def is_metatag?(tag) def is_metatag?(tag)
has_metatag?(tag, *METATAGS) has_metatag?(tag, *PostQueryBuilder::METATAGS)
end end
def is_negated_tag?(tag) def is_negated_tag?(tag)
@@ -448,357 +249,9 @@ class Tag < ApplicationRecord
def has_metatag?(tags, *metatags) def has_metatag?(tags, *metatags)
return nil if tags.blank? return nil if tags.blank?
tags = scan_query(tags.to_str) if tags.respond_to?(:to_str) tags = PostQueryBuilder.scan_query(tags.to_str) if tags.respond_to?(:to_str)
tags.grep(/\A(?:#{metatags.map(&:to_s).join("|")}):(.+)\z/i) { $1 }.first tags.grep(/\A(?:#{metatags.map(&:to_s).join("|")}):(.+)\z/i) { $1 }.first
end end
def parse_query(query, options = {})
q = {}
q[:tag_count] = 0
q[:tags] = {
:related => [],
:include => [],
:exclude => []
}
scan_query(query).each do |token|
q[:tag_count] += 1 unless Danbooru.config.is_unlimited_tag?(token)
if token =~ /\A(#{METATAGS.join("|")}):(.+)\z/i
g1 = $1.downcase
g2 = $2
case g1
when "-user"
q[:uploader_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:uploader_id_neg] << user_id unless user_id.blank?
when "user"
user_id = User.name_to_id(g2)
q[:uploader_id] = user_id unless user_id.blank?
when "-approver"
if g2 == "none"
q[:approver_id] = "any"
elsif g2 == "any"
q[:approver_id] = "none"
else
q[:approver_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:approver_id_neg] << user_id unless user_id.blank?
end
when "approver"
if g2 == "none"
q[:approver_id] = "none"
elsif g2 == "any"
q[:approver_id] = "any"
else
user_id = User.name_to_id(g2)
q[:approver_id] = user_id unless user_id.blank?
end
when "flagger"
q[:flagger_ids] ||= []
if g2 == "none"
q[:flagger_ids] << "none"
elsif g2 == "any"
q[:flagger_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:flagger_ids] << user_id unless user_id.blank?
end
when "-flagger"
if g2 == "none"
q[:flagger_ids] ||= []
q[:flagger_ids] << "any"
elsif g2 == "any"
q[:flagger_ids] ||= []
q[:flagger_ids] << "none"
else
q[:flagger_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:flagger_ids_neg] << user_id unless user_id.blank?
end
when "appealer"
q[:appealer_ids] ||= []
if g2 == "none"
q[:appealer_ids] << "none"
elsif g2 == "any"
q[:appealer_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:appealer_ids] << user_id unless user_id.blank?
end
when "-appealer"
if g2 == "none"
q[:appealer_ids] ||= []
q[:appealer_ids] << "any"
elsif g2 == "any"
q[:appealer_ids] ||= []
q[:appealer_ids] << "none"
else
q[:appealer_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:appealer_ids_neg] << user_id unless user_id.blank?
end
when "commenter", "comm"
q[:commenter_ids] ||= []
if g2 == "none"
q[:commenter_ids] << "none"
elsif g2 == "any"
q[:commenter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:commenter_ids] << user_id unless user_id.blank?
end
when "noter"
q[:noter_ids] ||= []
if g2 == "none"
q[:noter_ids] << "none"
elsif g2 == "any"
q[:noter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:noter_ids] << user_id unless user_id.blank?
end
when "noteupdater"
q[:note_updater_ids] ||= []
user_id = User.name_to_id(g2)
q[:note_updater_ids] << user_id unless user_id.blank?
when "artcomm"
q[:artcomm_ids] ||= []
user_id = User.name_to_id(g2)
q[:artcomm_ids] << user_id unless user_id.blank?
when "disapproved"
q[:disapproved] ||= []
q[:disapproved] << g2
when "-disapproved"
q[:disapproved_neg] ||= []
q[:disapproved_neg] << g2
when "-pool"
q[:pool_neg] ||= []
q[:pool_neg] << g2
when "pool"
q[:pool] ||= []
q[:pool] << g2
when "ordpool"
q[:ordpool] = g2
when "-favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups_neg] ||= []
q[:favgroups_neg] << favgroup
when "favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups] ||= []
q[:favgroups] << favgroup
when "-fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:exclude] << "fav:#{User.name_to_id(g2)}"
when "fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{User.name_to_id(g2)}"
when "ordfav"
user_id = User.name_to_id(g2)
favuser = User.find(user_id)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{user_id}"
q[:ordfav] = user_id
when "search"
q[:saved_searches] ||= []
q[:saved_searches] << g2
when "md5"
q[:md5] = g2.downcase.split(/,/)
when "-rating"
q[:rating_negated] = g2.downcase
when "rating"
q[:rating] = g2.downcase
when "-locked"
q[:locked_negated] = g2.downcase
when "locked"
q[:locked] = g2.downcase
when "id"
q[:post_id] = parse_helper(g2)
when "-id"
q[:post_id_negated] = g2.to_i
when "width"
q[:width] = parse_helper(g2)
when "height"
q[:height] = parse_helper(g2)
when "mpixels"
q[:mpixels] = parse_helper_fudged(g2, :float)
when "ratio"
q[:ratio] = parse_helper(g2, :ratio)
when "score"
q[:score] = parse_helper(g2)
when "favcount"
q[:fav_count] = parse_helper(g2)
when "filesize"
q[:filesize] = parse_helper_fudged(g2, :filesize)
when "source"
q[:source] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "-source"
q[:source_neg] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "date"
q[:date] = parse_helper(g2, :date)
when "age"
q[:age] = reverse_parse_helper(parse_helper(g2, :age))
when "tagcount"
q[:post_tag_count] = parse_helper(g2)
when /(#{TagCategory.short_name_regex})tags/
q["#{TagCategory.short_name_mapping[$1]}_tag_count".to_sym] = parse_helper(g2)
when "parent"
q[:parent] = g2.downcase
when "-parent"
if g2.downcase == "none"
q[:parent] = "any"
elsif g2.downcase == "any"
q[:parent] = "none"
else
q[:parent_neg_ids] ||= []
q[:parent_neg_ids] << g2.downcase
end
when "child"
q[:child] = g2.downcase
when "order"
g2 = g2.downcase
order, suffix, _tail = g2.partition(/_(asc|desc)\z/i)
if order.in?(COUNT_METATAG_SYNONYMS)
g2 = order.singularize + "_count" + suffix
end
q[:order] = g2
when "limit"
# Do nothing. The controller takes care of it.
when "-status"
q[:status_neg] = g2.downcase
when "status"
q[:status] = g2.downcase
when "embedded"
q[:embedded] = g2.downcase
when "filetype"
q[:filetype] = g2.downcase
when "-filetype"
q[:filetype_neg] = g2.downcase
when "pixiv_id", "pixiv"
if g2.downcase == "any" || g2.downcase == "none"
q[:pixiv_id] = g2.downcase
else
q[:pixiv_id] = parse_helper(g2)
end
when "upvote"
if CurrentUser.user.is_admin?
q[:upvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:upvote] = CurrentUser.user
end
when "downvote"
if CurrentUser.user.is_admin?
q[:downvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:downvote] = CurrentUser.user
end
when *COUNT_METATAGS
q[g1.to_sym] = parse_helper(g2)
when *COUNT_METATAG_SYNONYMS
g1 = "#{g1.singularize}_count"
q[g1.to_sym] = parse_helper(g2)
end
else
parse_tag(token, q[:tags])
end
end
normalize_tags_in_query(q)
return q
end
def normalize_tags_in_query(query_hash)
query_hash[:tags][:exclude] = TagAlias.to_aliased(query_hash[:tags][:exclude])
query_hash[:tags][:include] = TagAlias.to_aliased(query_hash[:tags][:include])
query_hash[:tags][:related] = TagAlias.to_aliased(query_hash[:tags][:related])
end
end end
module SearchMethods module SearchMethods

View File

@@ -1430,7 +1430,7 @@ class PostTest < ActiveSupport::TestCase
# final should be <aaa>, <bbb>, <ddd>, <eee> # final should be <aaa>, <bbb>, <ddd>, <eee>
final_post = Post.find(post.id) final_post = Post.find(post.id)
assert_equal(%w(aaa bbb ddd eee), Tag.scan_tags(final_post.tag_string).sort) assert_equal(%w(aaa bbb ddd eee), PostQueryBuilder.scan_query(final_post.tag_string).sort)
end end
should "merge any tag changes that were made after loading the initial set of tags part 2" do should "merge any tag changes that were made after loading the initial set of tags part 2" do
@@ -1453,7 +1453,7 @@ class PostTest < ActiveSupport::TestCase
# final should be <aaa>, <bbb>, <ddd>, <eee> # final should be <aaa>, <bbb>, <ddd>, <eee>
final_post = Post.find(post.id) final_post = Post.find(post.id)
assert_equal(%w(aaa bbb ddd eee), Tag.scan_tags(final_post.tag_string).sort) assert_equal(%w(aaa bbb ddd eee), PostQueryBuilder.scan_query(final_post.tag_string).sort)
end end
should "merge any parent, source, and rating changes that were made after loading the initial set" do should "merge any parent, source, and rating changes that were made after loading the initial set" do

View File

@@ -93,37 +93,36 @@ class TagTest < ActiveSupport::TestCase
context "A tag parser" do context "A tag parser" do
should "scan a query" do should "scan a query" do
assert_equal(%w(aaa bbb), Tag.scan_query("aaa bbb")) assert_equal(%w(aaa bbb), PostQueryBuilder.scan_query("aaa bbb"))
assert_equal(%w(~AAa -BBB* -bbb*), Tag.scan_query("~AAa -BBB* -bbb*")) assert_equal(%w(~AAa -BBB* -bbb*), PostQueryBuilder.scan_query("~AAa -BBB* -bbb*"))
end end
should "not strip out valid characters when scanning" do should "not strip out valid characters when scanning" do
assert_equal(%w(aaa bbb), Tag.scan_tags("aaa bbb")) assert_equal(%w(aaa bbb), PostQueryBuilder.scan_query("aaa bbb"))
assert_equal(%w(favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%), Tag.scan_tags("favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%")) assert_equal(%w(favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%), PostQueryBuilder.scan_query("favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%"))
end end
should "cast values" do should "cast values" do
assert_equal(2048, Tag.parse_cast("2kb", :filesize)) assert_equal(2048, PostQueryBuilder.parse_cast("2kb", :filesize))
assert_equal(2097152, Tag.parse_cast("2m", :filesize)) assert_equal(2097152, PostQueryBuilder.parse_cast("2m", :filesize))
assert_nothing_raised {Tag.parse_cast("2009-01-01", :date)} assert_nothing_raised {PostQueryBuilder.parse_cast("2009-01-01", :date)}
assert_nothing_raised {Tag.parse_cast("1234", :integer)} assert_nothing_raised {PostQueryBuilder.parse_cast("1234", :integer)}
assert_nothing_raised {Tag.parse_cast("1234.56", :float)} assert_nothing_raised {PostQueryBuilder.parse_cast("1234.56", :float)}
end end
should "parse a query" do should "parse a query" do
tag1 = FactoryBot.create(:tag, :name => "abc") tag1 = FactoryBot.create(:tag, :name => "abc")
tag2 = FactoryBot.create(:tag, :name => "acb") tag2 = FactoryBot.create(:tag, :name => "acb")
assert_equal(["abc"], Tag.parse_query("md5:abc")[:md5]) assert_equal(["abc"], PostQueryBuilder.parse_query("md5:abc")[:md5])
assert_equal([:between, 1, 2], Tag.parse_query("id:1..2")[:post_id]) assert_equal([:between, 1, 2], PostQueryBuilder.parse_query("id:1..2")[:post_id])
assert_equal([:gte, 1], Tag.parse_query("id:1..")[:post_id]) assert_equal([:gte, 1], PostQueryBuilder.parse_query("id:1..")[:post_id])
assert_equal([:lte, 2], Tag.parse_query("id:..2")[:post_id]) assert_equal([:lte, 2], PostQueryBuilder.parse_query("id:..2")[:post_id])
assert_equal([:gt, 2], Tag.parse_query("id:>2")[:post_id]) assert_equal([:gt, 2], PostQueryBuilder.parse_query("id:>2")[:post_id])
assert_equal([:lt, 3], Tag.parse_query("id:<3")[:post_id]) assert_equal([:lt, 3], PostQueryBuilder.parse_query("id:<3")[:post_id])
assert_equal([:lt, 3], Tag.parse_query("ID:<3")[:post_id]) assert_equal([:lt, 3], PostQueryBuilder.parse_query("ID:<3")[:post_id])
Tag.expects(:normalize_tags_in_query).returns(nil) assert_equal(["acb"], PostQueryBuilder.parse_query("a*b")[:tags][:include])
assert_equal(["acb"], Tag.parse_query("a*b")[:tags][:include])
end end
should "parse single tags correctly" do should "parse single tags correctly" do
@@ -238,7 +237,7 @@ class TagTest < ActiveSupport::TestCase
should_not allow_value("東方").for(:name).on(:create) should_not allow_value("東方").for(:name).on(:create)
should_not allow_value("FAV:blah").for(:name).on(:create) should_not allow_value("FAV:blah").for(:name).on(:create)
metatags = Tag::METATAGS + TagCategory.mapping.keys metatags = PostQueryBuilder::METATAGS + TagCategory.mapping.keys
metatags.each do |metatag| metatags.each do |metatag|
should_not allow_value("#{metatag}:foo").for(:name).on(:create) should_not allow_value("#{metatag}:foo").for(:name).on(:create)
end end