search: move query parsing code from tag model to post query builder.

This commit is contained in:
evazion
2020-03-06 21:14:50 -06:00
parent 2e0ad42eca
commit 967d398c8e
13 changed files with 583 additions and 598 deletions

View File

@@ -25,7 +25,7 @@ module PostsHelper
return unless post_search_counts_enabled?
return unless params[:action] == "index" && params[:page].nil? && params[:tags].present?
tags = Tag.scan_query(params[:tags]).sort.join(" ")
tags = PostQueryBuilder.scan_query(params[:tags]).sort.join(" ")
sig = generate_reportbooru_signature("ps-#{tags}")
render "posts/partials/index/search_count", sig: sig
end
@@ -63,7 +63,7 @@ module PostsHelper
end
def show_tag_change_notice?
Tag.scan_query(params[:tags]).size == 1 && TagChangeNoticeService.get_forum_topic_id(params[:tags])
PostQueryBuilder.scan_query(params[:tags]).size == 1 && TagChangeNoticeService.get_forum_topic_id(params[:tags])
end
private

View File

@@ -3,9 +3,9 @@ import CurrentUser from './current_user'
let Autocomplete = {};
/* eslint-disable */
Autocomplete.METATAGS = <%= Tag::METATAGS.to_json.html_safe %>;
Autocomplete.METATAGS = <%= PostQueryBuilder::METATAGS.to_json.html_safe %>;
Autocomplete.TAG_CATEGORIES = <%= TagCategory.mapping.to_json.html_safe %>;
Autocomplete.ORDER_METATAGS = <%= Tag::ORDER_METATAGS.to_json.html_safe %>;
Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_safe %>;
/* eslint-enable */
Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|");

View File

@@ -6,8 +6,8 @@ class TagBatchChangeJob < ApplicationJob
def perform(antecedent, consequent, updater, updater_ip_addr)
raise Error.new("antecedent is missing") if antecedent.blank?
normalized_antecedent = TagAlias.to_aliased(::Tag.scan_tags(antecedent.mb_chars.downcase))
normalized_consequent = TagAlias.to_aliased(::Tag.scan_tags(consequent.mb_chars.downcase))
normalized_antecedent = TagAlias.to_aliased(PostQueryBuilder.scan_query(antecedent.mb_chars.downcase))
normalized_consequent = TagAlias.to_aliased(PostQueryBuilder.scan_query(consequent.mb_chars.downcase))
CurrentUser.without_safe_mode do
CurrentUser.scoped(updater, updater_ip_addr) do
@@ -30,7 +30,7 @@ class TagBatchChangeJob < ApplicationJob
end
def migrate_saved_searches(normalized_antecedent, normalized_consequent)
tags = Tag.scan_tags(normalized_antecedent.join(" "), strip_metatags: true)
tags = PostQueryBuilder.scan_query(normalized_antecedent.join(" "), strip_metatags: true)
# https://www.postgresql.org/docs/current/static/functions-array.html
saved_searches = SavedSearch.where("string_to_array(query, ' ') @> ARRAY[?]", tags)
@@ -53,7 +53,7 @@ class TagBatchChangeJob < ApplicationJob
begin
repl = user.blacklisted_tags.split(/\r\n|\r|\n/).map do |line|
list = Tag.scan_tags(line)
list = PostQueryBuilder.scan_query(line)
if (list & query).size != query.size
next line

View File

@@ -88,8 +88,8 @@ class AliasAndImplicationImporter
all
when :mass_update
all += Tag.scan_tags(token[1])
all += Tag.scan_tags(token[2])
all += PostQueryBuilder.scan_query(token[1])
all += PostQueryBuilder.scan_query(token[2])
all
when :change_category

View File

@@ -64,7 +64,7 @@ module Searchable
def where_array_count(attr, value)
relation = all
qualified_column = "cardinality(#{qualified_column_for(attr)})"
parsed_range = Tag.parse_helper(value, :integer)
parsed_range = PostQueryBuilder.parse_helper(value, :integer)
PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, relation)
end
@@ -96,7 +96,7 @@ module Searchable
column = column_for_attribute(attribute)
qualified_column = "#{table_name}.#{column.name}"
parsed_range = Tag.parse_helper(range, column.type)
parsed_range = PostQueryBuilder.parse_helper(range, column.type)
PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, self)
end
@@ -252,7 +252,7 @@ module Searchable
def apply_default_order(params)
if params[:order] == "custom"
parse_ids = Tag.parse_helper(params[:id])
parse_ids = PostQueryBuilder.parse_helper(params[:id])
if parse_ids[0] == :in
return find_ordered(parse_ids[1])
end

View File

@@ -1,4 +1,49 @@
class PostQueryBuilder
COUNT_METATAGS = %w[
comment_count deleted_comment_count active_comment_count
note_count deleted_note_count active_note_count
flag_count resolved_flag_count unresolved_flag_count
child_count deleted_child_count active_child_count
pool_count deleted_pool_count active_pool_count series_pool_count collection_pool_count
appeal_count approval_count replacement_count
]
# allow e.g. `deleted_comments` as a synonym for `deleted_comment_count`
COUNT_METATAG_SYNONYMS = COUNT_METATAGS.map { |str| str.delete_suffix("_count").pluralize }
METATAGS = %w[
-user user -approver approver commenter comm noter noteupdater artcomm
-pool pool ordpool -favgroup favgroup -fav fav ordfav md5 -rating rating
-locked locked width height mpixels ratio score favcount filesize source
-source id -id date age order limit -status status tagcount parent -parent
child pixiv_id pixiv search upvote downvote filetype -filetype flagger
-flagger appealer -appealer disapproved -disapproved embedded
] + TagCategory.short_name_list.map {|x| "#{x}tags"} + COUNT_METATAGS + COUNT_METATAG_SYNONYMS
ORDER_METATAGS = %w[
id id_desc
score score_asc
favcount favcount_asc
created_at created_at_asc
change change_asc
comment comment_asc
comment_bumped comment_bumped_asc
note note_asc
artcomm artcomm_asc
mpixels mpixels_asc
portrait landscape
filesize filesize_asc
tagcount tagcount_asc
rank
curated
modqueue
random
custom
] +
COUNT_METATAGS +
COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } +
TagCategory.short_name_list.flat_map { |str| ["#{str}tags", "#{str}tags_asc"] }
attr_accessor :query_string
def initialize(query_string)
@@ -81,7 +126,7 @@ class PostQueryBuilder
end
def table_for_metatag(metatag)
if metatag.in?(Tag::COUNT_METATAGS)
if metatag.in?(COUNT_METATAGS)
metatag[/(?<table>[a-z]+)_count\z/i, :table]
else
nil
@@ -111,7 +156,7 @@ class PostQueryBuilder
def build
unless query_string.is_a?(Hash)
q = Tag.parse_query(query_string)
q = PostQueryBuilder.parse_query(query_string)
end
relation = Post.all
@@ -140,7 +185,7 @@ class PostQueryBuilder
end
relation = add_range_relation(q[:post_tag_count], "posts.tag_count", relation)
Tag::COUNT_METATAGS.each do |column|
COUNT_METATAGS.each do |column|
relation = add_range_relation(q[column.to_sym], "posts.#{column}", relation)
end
@@ -582,7 +627,7 @@ class PostQueryBuilder
when "filesize_asc"
relation = relation.order("posts.file_size ASC")
when /\A(?<column>#{Tag::COUNT_METATAGS.join("|")})(_(?<direction>asc|desc))?\z/i
when /\A(?<column>#{COUNT_METATAGS.join("|")})(_(?<direction>asc|desc))?\z/i
column = $~[:column]
direction = $~[:direction] || "desc"
relation = relation.order(column => direction, :id => direction)
@@ -624,4 +669,492 @@ class PostQueryBuilder
relation
end
concerning :ParseMethods do
class_methods do
def scan_query(query, strip_metatags: false)
tagstr = query.to_s.gsub(/\u3000/, " ").strip
list = tagstr.scan(/-?source:".*?"/) || []
list += tagstr.gsub(/-?source:".*?"/, "").scan(/[^[:space:]]+/).uniq
list = list.map { |tag| tag.sub(/^[-~]/, "") } if strip_metatags
list
end
def normalize_query(query, normalize_aliases: true, sort: true)
tags = scan_query(query.to_s)
tags = tags.map { |t| Tag.normalize_name(t) }
tags = TagAlias.to_aliased(tags) if normalize_aliases
tags = tags.sort if sort
tags = tags.uniq
tags.join(" ")
end
def parse_query(query, options = {})
q = {}
q[:tag_count] = 0
q[:tags] = {
:related => [],
:include => [],
:exclude => []
}
scan_query(query).each do |token|
q[:tag_count] += 1 unless Danbooru.config.is_unlimited_tag?(token)
if token =~ /\A(#{METATAGS.join("|")}):(.+)\z/i
g1 = $1.downcase
g2 = $2
case g1
when "-user"
q[:uploader_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:uploader_id_neg] << user_id unless user_id.blank?
when "user"
user_id = User.name_to_id(g2)
q[:uploader_id] = user_id unless user_id.blank?
when "-approver"
if g2 == "none"
q[:approver_id] = "any"
elsif g2 == "any"
q[:approver_id] = "none"
else
q[:approver_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:approver_id_neg] << user_id unless user_id.blank?
end
when "approver"
if g2 == "none"
q[:approver_id] = "none"
elsif g2 == "any"
q[:approver_id] = "any"
else
user_id = User.name_to_id(g2)
q[:approver_id] = user_id unless user_id.blank?
end
when "flagger"
q[:flagger_ids] ||= []
if g2 == "none"
q[:flagger_ids] << "none"
elsif g2 == "any"
q[:flagger_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:flagger_ids] << user_id unless user_id.blank?
end
when "-flagger"
if g2 == "none"
q[:flagger_ids] ||= []
q[:flagger_ids] << "any"
elsif g2 == "any"
q[:flagger_ids] ||= []
q[:flagger_ids] << "none"
else
q[:flagger_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:flagger_ids_neg] << user_id unless user_id.blank?
end
when "appealer"
q[:appealer_ids] ||= []
if g2 == "none"
q[:appealer_ids] << "none"
elsif g2 == "any"
q[:appealer_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:appealer_ids] << user_id unless user_id.blank?
end
when "-appealer"
if g2 == "none"
q[:appealer_ids] ||= []
q[:appealer_ids] << "any"
elsif g2 == "any"
q[:appealer_ids] ||= []
q[:appealer_ids] << "none"
else
q[:appealer_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:appealer_ids_neg] << user_id unless user_id.blank?
end
when "commenter", "comm"
q[:commenter_ids] ||= []
if g2 == "none"
q[:commenter_ids] << "none"
elsif g2 == "any"
q[:commenter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:commenter_ids] << user_id unless user_id.blank?
end
when "noter"
q[:noter_ids] ||= []
if g2 == "none"
q[:noter_ids] << "none"
elsif g2 == "any"
q[:noter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:noter_ids] << user_id unless user_id.blank?
end
when "noteupdater"
q[:note_updater_ids] ||= []
user_id = User.name_to_id(g2)
q[:note_updater_ids] << user_id unless user_id.blank?
when "artcomm"
q[:artcomm_ids] ||= []
user_id = User.name_to_id(g2)
q[:artcomm_ids] << user_id unless user_id.blank?
when "disapproved"
q[:disapproved] ||= []
q[:disapproved] << g2
when "-disapproved"
q[:disapproved_neg] ||= []
q[:disapproved_neg] << g2
when "-pool"
q[:pool_neg] ||= []
q[:pool_neg] << g2
when "pool"
q[:pool] ||= []
q[:pool] << g2
when "ordpool"
q[:ordpool] = g2
when "-favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups_neg] ||= []
q[:favgroups_neg] << favgroup
when "favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups] ||= []
q[:favgroups] << favgroup
when "-fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:exclude] << "fav:#{User.name_to_id(g2)}"
when "fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{User.name_to_id(g2)}"
when "ordfav"
user_id = User.name_to_id(g2)
favuser = User.find(user_id)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{user_id}"
q[:ordfav] = user_id
when "search"
q[:saved_searches] ||= []
q[:saved_searches] << g2
when "md5"
q[:md5] = g2.downcase.split(/,/)
when "-rating"
q[:rating_negated] = g2.downcase
when "rating"
q[:rating] = g2.downcase
when "-locked"
q[:locked_negated] = g2.downcase
when "locked"
q[:locked] = g2.downcase
when "id"
q[:post_id] = parse_helper(g2)
when "-id"
q[:post_id_negated] = g2.to_i
when "width"
q[:width] = parse_helper(g2)
when "height"
q[:height] = parse_helper(g2)
when "mpixels"
q[:mpixels] = parse_helper_fudged(g2, :float)
when "ratio"
q[:ratio] = parse_helper(g2, :ratio)
when "score"
q[:score] = parse_helper(g2)
when "favcount"
q[:fav_count] = parse_helper(g2)
when "filesize"
q[:filesize] = parse_helper_fudged(g2, :filesize)
when "source"
q[:source] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "-source"
q[:source_neg] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "date"
q[:date] = parse_helper(g2, :date)
when "age"
q[:age] = reverse_parse_helper(parse_helper(g2, :age))
when "tagcount"
q[:post_tag_count] = parse_helper(g2)
when /(#{TagCategory.short_name_regex})tags/
q["#{TagCategory.short_name_mapping[$1]}_tag_count".to_sym] = parse_helper(g2)
when "parent"
q[:parent] = g2.downcase
when "-parent"
if g2.downcase == "none"
q[:parent] = "any"
elsif g2.downcase == "any"
q[:parent] = "none"
else
q[:parent_neg_ids] ||= []
q[:parent_neg_ids] << g2.downcase
end
when "child"
q[:child] = g2.downcase
when "order"
g2 = g2.downcase
order, suffix, _tail = g2.partition(/_(asc|desc)\z/i)
if order.in?(COUNT_METATAG_SYNONYMS)
g2 = order.singularize + "_count" + suffix
end
q[:order] = g2
when "limit"
# Do nothing. The controller takes care of it.
when "-status"
q[:status_neg] = g2.downcase
when "status"
q[:status] = g2.downcase
when "embedded"
q[:embedded] = g2.downcase
when "filetype"
q[:filetype] = g2.downcase
when "-filetype"
q[:filetype_neg] = g2.downcase
when "pixiv_id", "pixiv"
if g2.downcase == "any" || g2.downcase == "none"
q[:pixiv_id] = g2.downcase
else
q[:pixiv_id] = parse_helper(g2)
end
when "upvote"
if CurrentUser.user.is_admin?
q[:upvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:upvote] = CurrentUser.user
end
when "downvote"
if CurrentUser.user.is_admin?
q[:downvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:downvote] = CurrentUser.user
end
when *COUNT_METATAGS
q[g1.to_sym] = parse_helper(g2)
when *COUNT_METATAG_SYNONYMS
g1 = "#{g1.singularize}_count"
q[g1.to_sym] = parse_helper(g2)
end
else
parse_tag(token, q[:tags])
end
end
q[:tags][:exclude] = TagAlias.to_aliased(q[:tags][:exclude])
q[:tags][:include] = TagAlias.to_aliased(q[:tags][:include])
q[:tags][:related] = TagAlias.to_aliased(q[:tags][:related])
return q
end
def parse_tag(tag, output)
if tag[0] == "-" && tag.size > 1
output[:exclude] << tag[1..-1].mb_chars.downcase
elsif tag[0] == "~" && tag.size > 1
output[:include] << tag[1..-1].mb_chars.downcase
elsif tag =~ /\*/
matches = Tag.name_matches(tag).select("name").limit(Danbooru.config.tag_query_limit).order("post_count DESC").map(&:name)
matches = ["~no_matches~"] if matches.empty?
output[:include] += matches
else
output[:related] << tag.mb_chars.downcase
end
end
def parse_cast(object, type)
case type
when :integer
object.to_i
when :float
object.to_f
when :date, :datetime
Time.zone.parse(object) rescue nil
when :age
DurationParser.parse(object).ago
when :ratio
object =~ /\A(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)\Z/i
if $1 && $2.to_f != 0.0
($1.to_f / $2.to_f).round(2)
else
object.to_f.round(2)
end
when :filesize
object =~ /\A(\d+(?:\.\d*)?|\d*\.\d+)([kKmM]?)[bB]?\Z/
size = $1.to_f
unit = $2
conversion_factor = case unit
when /m/i
1024 * 1024
when /k/i
1024
else
1
end
(size * conversion_factor).to_i
end
end
def parse_helper(range, type = :integer)
# "1", "0.5", "5.", ".5":
# (-?(\d+(\.\d*)?|\d*\.\d+))
case range
when /\A(.+?)\.\.(.+)/
return [:between, parse_cast($1, type), parse_cast($2, type)]
when /\A<=(.+)/, /\A\.\.(.+)/
return [:lte, parse_cast($1, type)]
when /\A<(.+)/
return [:lt, parse_cast($1, type)]
when /\A>=(.+)/, /\A(.+)\.\.\Z/
return [:gte, parse_cast($1, type)]
when /\A>(.+)/
return [:gt, parse_cast($1, type)]
when /[, ]/
return [:in, range.split(/[, ]+/).map {|x| parse_cast(x, type)}]
else
return [:eq, parse_cast(range, type)]
end
end
def parse_helper_fudged(range, type)
result = parse_helper(range, type)
# Don't fudge the filesize when searching filesize:123b or filesize:123.
if result[0] == :eq && type == :filesize && range !~ /[km]b?\Z/i
result
elsif result[0] == :eq
new_min = (result[1] * 0.95).to_i
new_max = (result[1] * 1.05).to_i
[:between, new_min, new_max]
else
result
end
end
def reverse_parse_helper(array)
case array[0]
when :between
[:between, *array[1..-1].reverse]
when :lte
[:gte, *array[1..-1]]
when :lt
[:gt, *array[1..-1]]
when :gte
[:lte, *array[1..-1]]
when :gt
[:lt, *array[1..-1]]
else
array
end
end
end
end
end

View File

@@ -4,7 +4,7 @@ module PostSets
attr_reader :tag_array, :page, :raw, :random, :post_count, :format
def initialize(tags, page = 1, per_page = nil, raw: false, random: false, format: "html")
@tag_array = Tag.scan_query(tags)
@tag_array = PostQueryBuilder.scan_query(tags)
@page = page
@per_page = per_page
@raw = raw.to_s.truthy?

View File

@@ -21,7 +21,7 @@ class TagNameValidator < ActiveModel::EachValidator
record.errors[attribute] << "'#{value}' cannot contain non-printable characters"
when /[^[:ascii:]]/
record.errors[attribute] << "'#{value}' must consist of only ASCII characters"
when /\A(#{Tag::METATAGS.join("|")}):(.+)\z/i
when /\A(#{PostQueryBuilder::METATAGS.join("|")}):(.+)\z/i
record.errors[attribute] << "'#{value}' cannot begin with '#{$1}:'"
when /\A(#{Tag.categories.regexp}):(.+)\z/i
record.errors[attribute] << "'#{value}' cannot begin with '#{$1}:'"

View File

@@ -522,11 +522,11 @@ class Post < ApplicationRecord
module TagMethods
def tag_array
@tag_array ||= Tag.scan_tags(tag_string)
@tag_array ||= PostQueryBuilder.scan_query(tag_string)
end
def tag_array_was
@tag_array_was ||= Tag.scan_tags(tag_string_in_database.presence || tag_string_before_last_save || "")
@tag_array_was ||= PostQueryBuilder.scan_query(tag_string_in_database.presence || tag_string_before_last_save || "")
end
def tags
@@ -590,7 +590,7 @@ class Post < ApplicationRecord
# then try to merge the tag changes together.
current_tags = tag_array_was
new_tags = tag_array
old_tags = Tag.scan_tags(old_tag_string)
old_tags = PostQueryBuilder.scan_query(old_tag_string)
kept_tags = current_tags & new_tags
@removed_tags = old_tags - kept_tags
@@ -627,7 +627,7 @@ class Post < ApplicationRecord
end
def normalize_tags
normalized_tags = Tag.scan_tags(tag_string)
normalized_tags = PostQueryBuilder.scan_query(tag_string)
normalized_tags = apply_casesensitive_metatags(normalized_tags)
normalized_tags = normalized_tags.map(&:downcase)
normalized_tags = filter_metatags(normalized_tags)
@@ -1058,7 +1058,7 @@ class Post < ApplicationRecord
tags = tags.to_s
tags += " rating:s" if CurrentUser.safe_mode?
tags += " -status:deleted" if CurrentUser.hide_deleted_posts? && !Tag.has_metatag?(tags, "status", "-status")
tags = Tag.normalize_query(tags)
tags = PostQueryBuilder.normalize_query(tags)
# Optimize some cases. these are just estimates but at these
# quantities being off by a few hundred doesn't matter much

View File

@@ -139,18 +139,18 @@ class SavedSearch < ApplicationRecord
.where(user_id: user_id)
.labeled(label)
.pluck(:query)
.map {|x| Tag.normalize_query(x, sort: true)}
.map {|x| PostQueryBuilder.normalize_query(x, sort: true)}
.sort
.uniq
end
end
def normalized_query
Tag.normalize_query(query, sort: true)
PostQueryBuilder.normalize_query(query, sort: true)
end
def normalize_query
self.query = Tag.normalize_query(query, sort: false)
self.query = PostQueryBuilder.normalize_query(query, sort: false)
end
end

View File

@@ -1,51 +1,4 @@
class Tag < ApplicationRecord
COUNT_METATAGS = %w[
comment_count deleted_comment_count active_comment_count
note_count deleted_note_count active_note_count
flag_count resolved_flag_count unresolved_flag_count
child_count deleted_child_count active_child_count
pool_count deleted_pool_count active_pool_count series_pool_count collection_pool_count
appeal_count approval_count replacement_count
]
# allow e.g. `deleted_comments` as a synonym for `deleted_comment_count`
COUNT_METATAG_SYNONYMS = COUNT_METATAGS.map { |str| str.delete_suffix("_count").pluralize }
METATAGS = %w[
-user user -approver approver commenter comm noter noteupdater artcomm
-pool pool ordpool -favgroup favgroup -fav fav ordfav md5 -rating rating
-locked locked width height mpixels ratio score favcount filesize source
-source id -id date age order limit -status status tagcount parent -parent
child pixiv_id pixiv search upvote downvote filetype -filetype flagger
-flagger appealer -appealer disapproved -disapproved embedded
] + TagCategory.short_name_list.map {|x| "#{x}tags"} + COUNT_METATAGS + COUNT_METATAG_SYNONYMS
SUBQUERY_METATAGS = %w[commenter comm noter noteupdater artcomm flagger -flagger appealer -appealer]
ORDER_METATAGS = %w[
id id_desc
score score_asc
favcount favcount_asc
created_at created_at_asc
change change_asc
comment comment_asc
comment_bumped comment_bumped_asc
note note_asc
artcomm artcomm_asc
mpixels mpixels_asc
portrait landscape
filesize filesize_asc
tagcount tagcount_asc
rank
curated
modqueue
random
custom
] +
COUNT_METATAGS +
COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } +
TagCategory.short_name_list.flat_map { |str| ["#{str}tags", "#{str}tags_asc"] }
has_one :wiki_page, :foreign_key => "title", :primary_key => "name"
has_one :artist, :foreign_key => "name", :primary_key => "name"
has_one :antecedent_alias, -> {active}, :class_name => "TagAlias", :foreign_key => "antecedent_name", :primary_key => "name"
@@ -268,169 +221,17 @@ class Tag < ApplicationRecord
end
module ParseMethods
def normalize(query)
query.to_s.gsub(/\u3000/, " ").strip
end
def normalize_query(query, normalize_aliases: true, sort: true)
tags = Tag.scan_query(query.to_s)
tags = tags.map { |t| Tag.normalize_name(t) }
tags = TagAlias.to_aliased(tags) if normalize_aliases
tags = tags.sort if sort
tags = tags.uniq
tags.join(" ")
end
def scan_query(query)
tagstr = normalize(query)
list = tagstr.scan(/-?source:".*?"/) || []
list + tagstr.gsub(/-?source:".*?"/, "").scan(/[^[:space:]]+/).uniq
end
def scan_tags(tags, options = {})
tagstr = normalize(tags)
list = tagstr.scan(/source:".*?"/) || []
list += tagstr.gsub(/source:".*?"/, "").scan(/[^[:space:]]+/).uniq
if options[:strip_metatags]
list = list.map {|x| x.sub(/^[-~]/, "")}
end
list
end
def parse_cast(object, type)
case type
when :integer
object.to_i
when :float
object.to_f
when :date, :datetime
Time.zone.parse(object) rescue nil
when :age
DurationParser.parse(object).ago
when :ratio
object =~ /\A(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)\Z/i
if $1 && $2.to_f != 0.0
($1.to_f / $2.to_f).round(2)
else
object.to_f.round(2)
end
when :filesize
object =~ /\A(\d+(?:\.\d*)?|\d*\.\d+)([kKmM]?)[bB]?\Z/
size = $1.to_f
unit = $2
conversion_factor = case unit
when /m/i
1024 * 1024
when /k/i
1024
else
1
end
(size * conversion_factor).to_i
end
end
def parse_helper(range, type = :integer)
# "1", "0.5", "5.", ".5":
# (-?(\d+(\.\d*)?|\d*\.\d+))
case range
when /\A(.+?)\.\.(.+)/
return [:between, parse_cast($1, type), parse_cast($2, type)]
when /\A<=(.+)/, /\A\.\.(.+)/
return [:lte, parse_cast($1, type)]
when /\A<(.+)/
return [:lt, parse_cast($1, type)]
when /\A>=(.+)/, /\A(.+)\.\.\Z/
return [:gte, parse_cast($1, type)]
when /\A>(.+)/
return [:gt, parse_cast($1, type)]
when /[, ]/
return [:in, range.split(/[, ]+/).map {|x| parse_cast(x, type)}]
else
return [:eq, parse_cast(range, type)]
end
end
def parse_helper_fudged(range, type)
result = parse_helper(range, type)
# Don't fudge the filesize when searching filesize:123b or filesize:123.
if result[0] == :eq && type == :filesize && range !~ /[km]b?\Z/i
result
elsif result[0] == :eq
new_min = (result[1] * 0.95).to_i
new_max = (result[1] * 1.05).to_i
[:between, new_min, new_max]
else
result
end
end
def reverse_parse_helper(array)
case array[0]
when :between
[:between, *array[1..-1].reverse]
when :lte
[:gte, *array[1..-1]]
when :lt
[:gt, *array[1..-1]]
when :gte
[:lte, *array[1..-1]]
when :gt
[:lt, *array[1..-1]]
else
array
end
end
def parse_tag(tag, output)
if tag[0] == "-" && tag.size > 1
output[:exclude] << tag[1..-1].mb_chars.downcase
elsif tag[0] == "~" && tag.size > 1
output[:include] << tag[1..-1].mb_chars.downcase
elsif tag =~ /\*/
matches = Tag.name_matches(tag).select("name").limit(Danbooru.config.tag_query_limit).order("post_count DESC").map(&:name)
matches = ["~no_matches~"] if matches.empty?
output[:include] += matches
else
output[:related] << tag.mb_chars.downcase
end
end
# true if query is a single "simple" tag (not a metatag, negated tag, or wildcard tag).
def is_simple_tag?(query)
is_single_tag?(query) && !is_metatag?(query) && !is_negated_tag?(query) && !is_optional_tag?(query) && !is_wildcard_tag?(query)
end
def is_single_tag?(query)
scan_query(query).size == 1
PostQueryBuilder.scan_query(query).size == 1
end
def is_metatag?(tag)
has_metatag?(tag, *METATAGS)
has_metatag?(tag, *PostQueryBuilder::METATAGS)
end
def is_negated_tag?(tag)
@@ -448,357 +249,9 @@ class Tag < ApplicationRecord
def has_metatag?(tags, *metatags)
return nil if tags.blank?
tags = scan_query(tags.to_str) if tags.respond_to?(:to_str)
tags = PostQueryBuilder.scan_query(tags.to_str) if tags.respond_to?(:to_str)
tags.grep(/\A(?:#{metatags.map(&:to_s).join("|")}):(.+)\z/i) { $1 }.first
end
def parse_query(query, options = {})
q = {}
q[:tag_count] = 0
q[:tags] = {
:related => [],
:include => [],
:exclude => []
}
scan_query(query).each do |token|
q[:tag_count] += 1 unless Danbooru.config.is_unlimited_tag?(token)
if token =~ /\A(#{METATAGS.join("|")}):(.+)\z/i
g1 = $1.downcase
g2 = $2
case g1
when "-user"
q[:uploader_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:uploader_id_neg] << user_id unless user_id.blank?
when "user"
user_id = User.name_to_id(g2)
q[:uploader_id] = user_id unless user_id.blank?
when "-approver"
if g2 == "none"
q[:approver_id] = "any"
elsif g2 == "any"
q[:approver_id] = "none"
else
q[:approver_id_neg] ||= []
user_id = User.name_to_id(g2)
q[:approver_id_neg] << user_id unless user_id.blank?
end
when "approver"
if g2 == "none"
q[:approver_id] = "none"
elsif g2 == "any"
q[:approver_id] = "any"
else
user_id = User.name_to_id(g2)
q[:approver_id] = user_id unless user_id.blank?
end
when "flagger"
q[:flagger_ids] ||= []
if g2 == "none"
q[:flagger_ids] << "none"
elsif g2 == "any"
q[:flagger_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:flagger_ids] << user_id unless user_id.blank?
end
when "-flagger"
if g2 == "none"
q[:flagger_ids] ||= []
q[:flagger_ids] << "any"
elsif g2 == "any"
q[:flagger_ids] ||= []
q[:flagger_ids] << "none"
else
q[:flagger_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:flagger_ids_neg] << user_id unless user_id.blank?
end
when "appealer"
q[:appealer_ids] ||= []
if g2 == "none"
q[:appealer_ids] << "none"
elsif g2 == "any"
q[:appealer_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:appealer_ids] << user_id unless user_id.blank?
end
when "-appealer"
if g2 == "none"
q[:appealer_ids] ||= []
q[:appealer_ids] << "any"
elsif g2 == "any"
q[:appealer_ids] ||= []
q[:appealer_ids] << "none"
else
q[:appealer_ids_neg] ||= []
user_id = User.name_to_id(g2)
q[:appealer_ids_neg] << user_id unless user_id.blank?
end
when "commenter", "comm"
q[:commenter_ids] ||= []
if g2 == "none"
q[:commenter_ids] << "none"
elsif g2 == "any"
q[:commenter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:commenter_ids] << user_id unless user_id.blank?
end
when "noter"
q[:noter_ids] ||= []
if g2 == "none"
q[:noter_ids] << "none"
elsif g2 == "any"
q[:noter_ids] << "any"
else
user_id = User.name_to_id(g2)
q[:noter_ids] << user_id unless user_id.blank?
end
when "noteupdater"
q[:note_updater_ids] ||= []
user_id = User.name_to_id(g2)
q[:note_updater_ids] << user_id unless user_id.blank?
when "artcomm"
q[:artcomm_ids] ||= []
user_id = User.name_to_id(g2)
q[:artcomm_ids] << user_id unless user_id.blank?
when "disapproved"
q[:disapproved] ||= []
q[:disapproved] << g2
when "-disapproved"
q[:disapproved_neg] ||= []
q[:disapproved_neg] << g2
when "-pool"
q[:pool_neg] ||= []
q[:pool_neg] << g2
when "pool"
q[:pool] ||= []
q[:pool] << g2
when "ordpool"
q[:ordpool] = g2
when "-favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups_neg] ||= []
q[:favgroups_neg] << favgroup
when "favgroup"
favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user)
raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user)
q[:favgroups] ||= []
q[:favgroups] << favgroup
when "-fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:exclude] << "fav:#{User.name_to_id(g2)}"
when "fav"
favuser = User.find_by_name(g2)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{User.name_to_id(g2)}"
when "ordfav"
user_id = User.name_to_id(g2)
favuser = User.find(user_id)
if favuser.hide_favorites?
raise User::PrivilegeError.new
end
q[:tags][:related] << "fav:#{user_id}"
q[:ordfav] = user_id
when "search"
q[:saved_searches] ||= []
q[:saved_searches] << g2
when "md5"
q[:md5] = g2.downcase.split(/,/)
when "-rating"
q[:rating_negated] = g2.downcase
when "rating"
q[:rating] = g2.downcase
when "-locked"
q[:locked_negated] = g2.downcase
when "locked"
q[:locked] = g2.downcase
when "id"
q[:post_id] = parse_helper(g2)
when "-id"
q[:post_id_negated] = g2.to_i
when "width"
q[:width] = parse_helper(g2)
when "height"
q[:height] = parse_helper(g2)
when "mpixels"
q[:mpixels] = parse_helper_fudged(g2, :float)
when "ratio"
q[:ratio] = parse_helper(g2, :ratio)
when "score"
q[:score] = parse_helper(g2)
when "favcount"
q[:fav_count] = parse_helper(g2)
when "filesize"
q[:filesize] = parse_helper_fudged(g2, :filesize)
when "source"
q[:source] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "-source"
q[:source_neg] = g2.gsub(/\A"(.*)"\Z/, '\1')
when "date"
q[:date] = parse_helper(g2, :date)
when "age"
q[:age] = reverse_parse_helper(parse_helper(g2, :age))
when "tagcount"
q[:post_tag_count] = parse_helper(g2)
when /(#{TagCategory.short_name_regex})tags/
q["#{TagCategory.short_name_mapping[$1]}_tag_count".to_sym] = parse_helper(g2)
when "parent"
q[:parent] = g2.downcase
when "-parent"
if g2.downcase == "none"
q[:parent] = "any"
elsif g2.downcase == "any"
q[:parent] = "none"
else
q[:parent_neg_ids] ||= []
q[:parent_neg_ids] << g2.downcase
end
when "child"
q[:child] = g2.downcase
when "order"
g2 = g2.downcase
order, suffix, _tail = g2.partition(/_(asc|desc)\z/i)
if order.in?(COUNT_METATAG_SYNONYMS)
g2 = order.singularize + "_count" + suffix
end
q[:order] = g2
when "limit"
# Do nothing. The controller takes care of it.
when "-status"
q[:status_neg] = g2.downcase
when "status"
q[:status] = g2.downcase
when "embedded"
q[:embedded] = g2.downcase
when "filetype"
q[:filetype] = g2.downcase
when "-filetype"
q[:filetype_neg] = g2.downcase
when "pixiv_id", "pixiv"
if g2.downcase == "any" || g2.downcase == "none"
q[:pixiv_id] = g2.downcase
else
q[:pixiv_id] = parse_helper(g2)
end
when "upvote"
if CurrentUser.user.is_admin?
q[:upvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:upvote] = CurrentUser.user
end
when "downvote"
if CurrentUser.user.is_admin?
q[:downvote] = User.find_by_name(g2)
elsif CurrentUser.user.is_voter?
q[:downvote] = CurrentUser.user
end
when *COUNT_METATAGS
q[g1.to_sym] = parse_helper(g2)
when *COUNT_METATAG_SYNONYMS
g1 = "#{g1.singularize}_count"
q[g1.to_sym] = parse_helper(g2)
end
else
parse_tag(token, q[:tags])
end
end
normalize_tags_in_query(q)
return q
end
def normalize_tags_in_query(query_hash)
query_hash[:tags][:exclude] = TagAlias.to_aliased(query_hash[:tags][:exclude])
query_hash[:tags][:include] = TagAlias.to_aliased(query_hash[:tags][:include])
query_hash[:tags][:related] = TagAlias.to_aliased(query_hash[:tags][:related])
end
end
module SearchMethods

View File

@@ -1430,7 +1430,7 @@ class PostTest < ActiveSupport::TestCase
# final should be <aaa>, <bbb>, <ddd>, <eee>
final_post = Post.find(post.id)
assert_equal(%w(aaa bbb ddd eee), Tag.scan_tags(final_post.tag_string).sort)
assert_equal(%w(aaa bbb ddd eee), PostQueryBuilder.scan_query(final_post.tag_string).sort)
end
should "merge any tag changes that were made after loading the initial set of tags part 2" do
@@ -1453,7 +1453,7 @@ class PostTest < ActiveSupport::TestCase
# final should be <aaa>, <bbb>, <ddd>, <eee>
final_post = Post.find(post.id)
assert_equal(%w(aaa bbb ddd eee), Tag.scan_tags(final_post.tag_string).sort)
assert_equal(%w(aaa bbb ddd eee), PostQueryBuilder.scan_query(final_post.tag_string).sort)
end
should "merge any parent, source, and rating changes that were made after loading the initial set" do

View File

@@ -93,37 +93,36 @@ class TagTest < ActiveSupport::TestCase
context "A tag parser" do
should "scan a query" do
assert_equal(%w(aaa bbb), Tag.scan_query("aaa bbb"))
assert_equal(%w(~AAa -BBB* -bbb*), Tag.scan_query("~AAa -BBB* -bbb*"))
assert_equal(%w(aaa bbb), PostQueryBuilder.scan_query("aaa bbb"))
assert_equal(%w(~AAa -BBB* -bbb*), PostQueryBuilder.scan_query("~AAa -BBB* -bbb*"))
end
should "not strip out valid characters when scanning" do
assert_equal(%w(aaa bbb), Tag.scan_tags("aaa bbb"))
assert_equal(%w(favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%), Tag.scan_tags("favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%"))
assert_equal(%w(aaa bbb), PostQueryBuilder.scan_query("aaa bbb"))
assert_equal(%w(favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%), PostQueryBuilder.scan_query("favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%"))
end
should "cast values" do
assert_equal(2048, Tag.parse_cast("2kb", :filesize))
assert_equal(2097152, Tag.parse_cast("2m", :filesize))
assert_nothing_raised {Tag.parse_cast("2009-01-01", :date)}
assert_nothing_raised {Tag.parse_cast("1234", :integer)}
assert_nothing_raised {Tag.parse_cast("1234.56", :float)}
assert_equal(2048, PostQueryBuilder.parse_cast("2kb", :filesize))
assert_equal(2097152, PostQueryBuilder.parse_cast("2m", :filesize))
assert_nothing_raised {PostQueryBuilder.parse_cast("2009-01-01", :date)}
assert_nothing_raised {PostQueryBuilder.parse_cast("1234", :integer)}
assert_nothing_raised {PostQueryBuilder.parse_cast("1234.56", :float)}
end
should "parse a query" do
tag1 = FactoryBot.create(:tag, :name => "abc")
tag2 = FactoryBot.create(:tag, :name => "acb")
assert_equal(["abc"], Tag.parse_query("md5:abc")[:md5])
assert_equal([:between, 1, 2], Tag.parse_query("id:1..2")[:post_id])
assert_equal([:gte, 1], Tag.parse_query("id:1..")[:post_id])
assert_equal([:lte, 2], Tag.parse_query("id:..2")[:post_id])
assert_equal([:gt, 2], Tag.parse_query("id:>2")[:post_id])
assert_equal([:lt, 3], Tag.parse_query("id:<3")[:post_id])
assert_equal([:lt, 3], Tag.parse_query("ID:<3")[:post_id])
assert_equal(["abc"], PostQueryBuilder.parse_query("md5:abc")[:md5])
assert_equal([:between, 1, 2], PostQueryBuilder.parse_query("id:1..2")[:post_id])
assert_equal([:gte, 1], PostQueryBuilder.parse_query("id:1..")[:post_id])
assert_equal([:lte, 2], PostQueryBuilder.parse_query("id:..2")[:post_id])
assert_equal([:gt, 2], PostQueryBuilder.parse_query("id:>2")[:post_id])
assert_equal([:lt, 3], PostQueryBuilder.parse_query("id:<3")[:post_id])
assert_equal([:lt, 3], PostQueryBuilder.parse_query("ID:<3")[:post_id])
Tag.expects(:normalize_tags_in_query).returns(nil)
assert_equal(["acb"], Tag.parse_query("a*b")[:tags][:include])
assert_equal(["acb"], PostQueryBuilder.parse_query("a*b")[:tags][:include])
end
should "parse single tags correctly" do
@@ -238,7 +237,7 @@ class TagTest < ActiveSupport::TestCase
should_not allow_value("東方").for(:name).on(:create)
should_not allow_value("FAV:blah").for(:name).on(:create)
metatags = Tag::METATAGS + TagCategory.mapping.keys
metatags = PostQueryBuilder::METATAGS + TagCategory.mapping.keys
metatags.each do |metatag|
should_not allow_value("#{metatag}:foo").for(:name).on(:create)
end