jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

search: move query parsing code from tag model to post query builder.

Browse Source
This commit is contained in:
evazion
2020-03-06 21:14:50 -06:00
parent 2e0ad42eca
commit 967d398c8e
13 changed files with 583 additions and 598 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/models/post.rb
View File

@@ -522,11 +522,11 @@ class Post < ApplicationRecord
module TagMethods
def tag_array
@tag_array ||= Tag.scan_tags(tag_string)
@tag_array ||= PostQueryBuilder.scan_query(tag_string)
end
def tag_array_was
@tag_array_was ||= Tag.scan_tags(tag_string_in_database.presence || tag_string_before_last_save || "")
@tag_array_was ||= PostQueryBuilder.scan_query(tag_string_in_database.presence || tag_string_before_last_save || "")
end
def tags
@@ -590,7 +590,7 @@ class Post < ApplicationRecord
# then try to merge the tag changes together.
current_tags = tag_array_was
new_tags = tag_array
old_tags = Tag.scan_tags(old_tag_string)
old_tags = PostQueryBuilder.scan_query(old_tag_string)
kept_tags = current_tags & new_tags
@removed_tags = old_tags - kept_tags
@@ -627,7 +627,7 @@ class Post < ApplicationRecord
end
def normalize_tags
normalized_tags = Tag.scan_tags(tag_string)
normalized_tags = PostQueryBuilder.scan_query(tag_string)
normalized_tags = apply_casesensitive_metatags(normalized_tags)
normalized_tags = normalized_tags.map(&:downcase)
normalized_tags = filter_metatags(normalized_tags)
@@ -1058,7 +1058,7 @@ class Post < ApplicationRecord
tags = tags.to_s
tags += " rating:s" if CurrentUser.safe_mode?
tags += " -status:deleted" if CurrentUser.hide_deleted_posts? && !Tag.has_metatag?(tags, "status", "-status")
tags = Tag.normalize_query(tags)
tags = PostQueryBuilder.normalize_query(tags)
# Optimize some cases. these are just estimates but at these
# quantities being off by a few hundred doesn't matter much

6
app/models/saved_search.rb
View File

@@ -139,18 +139,18 @@ class SavedSearch < ApplicationRecord
.where(user_id: user_id)
.labeled(label)
.pluck(:query)
.map {|x| Tag.normalize_query(x, sort: true)}
.map {|x| PostQueryBuilder.normalize_query(x, sort: true)}
.sort
.uniq
end
end
def normalized_query
Tag.normalize_query(query, sort: true)
PostQueryBuilder.normalize_query(query, sort: true)
end
def normalize_query
self.query = Tag.normalize_query(query, sort: false)
self.query = PostQueryBuilder.normalize_query(query, sort: false)
end
end

553
app/models/tag.rb
View File

@@ -1,51 +1,4 @@
class Tag < ApplicationRecord
COUNT_METATAGS = %w[
comment_count deleted_comment_count active_comment_count
note_count deleted_note_count active_note_count
flag_count resolved_flag_count unresolved_flag_count
child_count deleted_child_count active_child_count
pool_count deleted_pool_count active_pool_count series_pool_count collection_pool_count
appeal_count approval_count replacement_count
]
# allow e.g. `deleted_comments` as a synonym for `deleted_comment_count`
COUNT_METATAG_SYNONYMS = COUNT_METATAGS.map { |str| str.delete_suffix("_count").pluralize }
METATAGS = %w[
-user user -approver approver commenter comm noter noteupdater artcomm
-pool pool ordpool -favgroup favgroup -fav fav ordfav md5 -rating rating
-locked locked width height mpixels ratio score favcount filesize source
-source id -id date age order limit -status status tagcount parent -parent
child pixiv_id pixiv search upvote downvote filetype -filetype flagger
-flagger appealer -appealer disapproved -disapproved embedded
] + TagCategory.short_name_list.map {|x| "#{x}tags"} + COUNT_METATAGS + COUNT_METATAG_SYNONYMS
SUBQUERY_METATAGS = %w[commenter comm noter noteupdater artcomm flagger -flagger appealer -appealer]
ORDER_METATAGS = %w[
id id_desc
score score_asc
favcount favcount_asc
created_at created_at_asc
change change_asc
comment comment_asc
comment_bumped comment_bumped_asc
note note_asc
artcomm artcomm_asc
mpixels mpixels_asc
portrait landscape
filesize filesize_asc
tagcount tagcount_asc
rank
curated
modqueue
random
custom
] +
COUNT_METATAGS +
COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } +
TagCategory.short_name_list.flat_map { |str| ["#{str}tags", "#{str}tags_asc"] }
has_one :wiki_page, :foreign_key => "title", :primary_key => "name"
has_one :artist, :foreign_key => "name", :primary_key => "name"
has_one :antecedent_alias, -> {active}, :class_name => "TagAlias", :foreign_key => "antecedent_name", :primary_key => "name"
@@ -268,169 +221,17 @@ class Tag < ApplicationRecord
end
module ParseMethods
def normalize(query)
query.to_s.gsub(/\u3000/, " ").strip
end
def normalize_query(query, normalize_aliases: true, sort: true)
tags = Tag.scan_query(query.to_s)
tags = tags.map { |t| Tag.normalize_name(t) }
tags = TagAlias.to_aliased(tags) if normalize_aliases
tags = tags.sort if sort
tags = tags.uniq
tags.join(" ")
end
def scan_query(query)
tagstr = normalize(query)
list = tagstr.scan(/-?source:".*?"/) || []
list + tagstr.gsub(/-?source:".*?"/, "").scan(/[^[:space:]]+/).uniq
end
def scan_tags(tags, options = {})
tagstr = normalize(tags)
list = tagstr.scan(/source:".*?"/) || []
list += tagstr.gsub(/source:".*?"/, "").scan(/[^[:space:]]+/).uniq
if options[:strip_metatags]
list = list.map {|x| x.sub(/^[-~]/, "")}
end
list
end
def parse_cast(object, type)
case type
when :integer
object.to_i
when :float
object.to_f
when :date, :datetime
Time.zone.parse(object) rescue nil
when :age
DurationParser.parse(object).ago
when :ratio
object =~ /\A(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)\Z/i
if $1 && $2.to_f != 0.0
($1.to_f / $2.to_f).round(2)
else
object.to_f.round(2)
end
when :filesize
object =~ /\A(\d+(?:\.\d*)?|\d*\.\d+)([kKmM]?)[bB]?\Z/
size = $1.to_f
unit = $2
conversion_factor = case unit
when /m/i
1024 * 1024
when /k/i
1024
else
1
end
(size * conversion_factor).to_i
end
end
def parse_helper(range, type = :integer)
# "1", "0.5", "5.", ".5":
# (-?(\d+(\.\d*)?|\d*\.\d+))
case range
when /\A(.+?)\.\.(.+)/
return [:between, parse_cast($1, type), parse_cast($2, type)]
when /\A<=(.+)/, /\A\.\.(.+)/
return [:lte, parse_cast($1, type)]
when /\A<(.+)/
return [:lt, parse_cast($1, type)]
when /\A>=(.+)/, /\A(.+)\.\.\Z/
return [:gte, parse_cast($1, type)]
when /\A>(.+)/
return [:gt, parse_cast($1, type)]
when /[, ]/
return [:in, range.split(/[, ]+/).map {|x| parse_cast(x, type)}]
else
return [:eq, parse_cast(range, type)]
end
end
def parse_helper_fudged(range, type)
result = parse_helper(range, type)
# Don't fudge the filesize when searching filesize:123b or filesize:123.
if result[0] == :eq && type == :filesize && range !~ /[km]b?\Z/i
result
elsif result[0] == :eq
new_min = (result[1] * 0.95).to_i
new_max = (result[1] * 1.05).to_i
[:between, new_min, new_max]
else
result
end
end
def reverse_parse_helper(array)
case array[0]
when :between
[:between, *array[1..-1].reverse]
when :lte
[:gte, *array[1..-1]]
when :lt
[:gt, *array[1..-1]]
when :gte
[:lte, *array[1..-1]]
when :gt
[:lt, *array[1..-1]]
else
array
end
end
def parse_tag(tag, output)
if tag[0] == "-" && tag.size > 1
output[:exclude] << tag[1..-1].mb_chars.downcase
elsif tag[0] == "~" && tag.size > 1
output[:include] << tag[1..-1].mb_chars.downcase
elsif tag =~ /\*/
matches = Tag.name_matches(tag).select("name").limit(Danbooru.config.tag_query_limit).order("post_count DESC").map(&:name)
matches = ["~no_matches~"] if matches.empty?
output[:include] += matches
else
output[:related] << tag.mb_chars.downcase
end
end
# true if query is a single "simple" tag (not a metatag, negated tag, or wildcard tag).
def is_simple_tag?(query)
is_single_tag?(query) && !is_metatag?(query) && !is_negated_tag?(query) && !is_optional_tag?(query) && !is_wildcard_tag?(query)
end
def is_single_tag?(query)
scan_query(query).size == 1
PostQueryBuilder.scan_query(query).size == 1
end
def is_metatag?(tag)
has_metatag?(tag, *METATAGS)
has_metatag?(tag, *PostQueryBuilder::METATAGS)
end
def is_negated_tag?(tag)
@@ -448,357 +249,9 @@ class Tag < ApplicationRecord
def has_metatag?(tags, *metatags)
return nil if tags.blank?
tags = scan_query(tags.to_str) if tags.respond_to?(:to_str)
tags = PostQueryBuilder.scan_query(tags.to_str) if tags.respond_to?(:to_str)
tags.grep(/\A(?:#{metatags.map(&:to_s).join("|")}):(.+)\z/i) { $1 }.first
end
def parse_query(query, options = {})
q = {}
q[:tag_count] = 0
q[:tags] = {
:related => [],
:include => [],
:exclude => []
}
scan_query(query).each do |token|
q[:tag_count] += 1 unless Danbooru.config.is_unlimited_tag?(token)
if token =~ /\A(#{METATAGS.join("|")}):(.+)\z/i
g1 = $1.downcase
g2 = $2
case g1
when "-user"
q[:uploader_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:uploader_id_neg] << user_id unless user_id.blank?
when "user"
user_id = User.name_to_id(g2)
q[:uploader_id] = user_id unless user_id.blank?
when "-approver"
if g2 == "none"
q[:approver_id] = "any"
elsif g2 == "any"
q[:approver_id] = "none"
else
q[:approver_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:approver_id_neg] << user_id unless user_id.blank?
end
when "approver"
if g2 == "none"
q[:approver_id] = "none"
elsif g2 == "any"
q[:approver_id] = "any"
else
user_id = User.name_to_id(g2)
q[:approver_id] = user_id unless user_id.blank?
end
when "flagger"
q[:flagger_ids] ||= []
if g2 == "none"
q[:flagger_ids] << "none"
elsif g2 == "any"
q[:flagger_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:flagger_ids] << user_id unless user_id.blank?
end
when "-flagger"
if g2 == "none"
q[:flagger_ids] ||= []
q[:flagger_ids] << "any"
elsif g2 == "any"
q[:flagger_ids] ||= []
q[:flagger_ids] << "none"
else
q[:flagger_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:flagger_ids_neg] << user_id unless user_id.blank?
end
when "appealer"
q[:appealer_ids] ||= []
if g2 == "none"
q[:appealer_ids] << "none"
elsif g2 == "any"
q[:appealer_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:appealer_ids] << user_id unless user_id.blank?
end
when "-appealer"
if g2 == "none"
q[:appealer_ids] ||= []
q[:appealer_ids] << "any"
elsif g2 == "any"
q[:appealer_ids] ||= []
q[:appealer_ids] << "none"
else
q[:appealer_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:appealer_ids_neg] << user_id unless user_id.blank?
end
when "commenter", "comm"
q[:commenter_ids] ||= []
if g2 == "none"
q[:commenter_ids] << "none"
elsif g2 == "any"
q[:commenter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:commenter_ids] << user_id unless user_id.blank?
end
when "noter"
q[:noter_ids] ||= []
if g2 == "none"
q[:noter_ids] << "none"
elsif g2 == "any"
q[:noter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:noter_ids] << user_id unless user_id.blank?
end
when "noteupdater"
q[:note_updater_ids] ||= []
user_id = User.name_to_id(g2)
q[:note_updater_ids] << user_id unless user_id.blank?
when "artcomm"
q[:artcomm_ids] ||= []
user_id = User.name_to_id(g2)
q[:artcomm_ids] << user_id unless user_id.blank?
when "disapproved"
q[:disapproved] ||= []
q[:disapproved] << g2
when "-disapproved"
q[:disapproved_neg] ||= []
q[:disapproved_neg] << g2
when "-pool"
q[:pool_neg] ||= []
q[:pool_neg] << g2
when "pool"
q[:pool] ||= []
q[:pool] << g2
when "ordpool"
q[:ordpool] = g2
when "-favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups_neg] ||= []
q[:favgroups_neg] << favgroup
when "favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups] ||= []
q[:favgroups] << favgroup
when "-fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:exclude] << "fav:#{User.name_to_id(g2)}"
when "fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{User.name_to_id(g2)}"
when "ordfav"
user_id = User.name_to_id(g2)
favuser = User.find(user_id)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{user_id}"
q[:ordfav] = user_id
when "search"
q[:saved_searches] ||= []
q[:saved_searches] << g2
when "md5"
q[:md5] = g2.downcase.split(/,/)
when "-rating"
q[:rating_negated] = g2.downcase
when "rating"
q[:rating] = g2.downcase
when "-locked"
q[:locked_negated] = g2.downcase
when "locked"
q[:locked] = g2.downcase
when "id"
q[:post_id] = parse_helper(g2)
when "-id"
q[:post_id_negated] = g2.to_i
when "width"
q[:width] = parse_helper(g2)
when "height"
q[:height] = parse_helper(g2)
when "mpixels"
q[:mpixels] = parse_helper_fudged(g2, :float)
when "ratio"
q[:ratio] = parse_helper(g2, :ratio)
when "score"
q[:score] = parse_helper(g2)
when "favcount"
q[:fav_count] = parse_helper(g2)
when "filesize"
q[:filesize] = parse_helper_fudged(g2, :filesize)
when "source"
q[:source] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "-source"
q[:source_neg] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "date"
q[:date] = parse_helper(g2, :date)
when "age"
q[:age] = reverse_parse_helper(parse_helper(g2, :age))
when "tagcount"
q[:post_tag_count] = parse_helper(g2)
when /(#{TagCategory.short_name_regex})tags/
q["#{TagCategory.short_name_mapping[$1]}_tag_count".to_sym] = parse_helper(g2)
when "parent"
q[:parent] = g2.downcase
when "-parent"
if g2.downcase == "none"
q[:parent] = "any"
elsif g2.downcase == "any"
q[:parent] = "none"
else
q[:parent_neg_ids] ||= []
q[:parent_neg_ids] << g2.downcase
end
when "child"
q[:child] = g2.downcase
when "order"
g2 = g2.downcase
order, suffix, _tail = g2.partition(/_(asc|desc)\z/i)
if order.in?(COUNT_METATAG_SYNONYMS)
g2 = order.singularize + "_count" + suffix
end
q[:order] = g2
when "limit"
# Do nothing. The controller takes care of it.
when "-status"
q[:status_neg] = g2.downcase
when "status"
q[:status] = g2.downcase
when "embedded"
q[:embedded] = g2.downcase
when "filetype"
q[:filetype] = g2.downcase
when "-filetype"
q[:filetype_neg] = g2.downcase
when "pixiv_id", "pixiv"
if g2.downcase == "any" || g2.downcase == "none"
q[:pixiv_id] = g2.downcase
else
q[:pixiv_id] = parse_helper(g2)
end
when "upvote"
if CurrentUser.user.is_admin?
q[:upvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:upvote] = CurrentUser.user
end
when "downvote"
if CurrentUser.user.is_admin?
q[:downvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:downvote] = CurrentUser.user
end
when *COUNT_METATAGS
q[g1.to_sym] = parse_helper(g2)
when *COUNT_METATAG_SYNONYMS
g1 = "#{g1.singularize}_count"
q[g1.to_sym] = parse_helper(g2)
end
else
parse_tag(token, q[:tags])
end
end
normalize_tags_in_query(q)
return q
end
def normalize_tags_in_query(query_hash)
query_hash[:tags][:exclude] = TagAlias.to_aliased(query_hash[:tags][:exclude])
query_hash[:tags][:include] = TagAlias.to_aliased(query_hash[:tags][:include])
query_hash[:tags][:related] = TagAlias.to_aliased(query_hash[:tags][:related])
end
end
module SearchMethods