From 967d398c8e8763bbcf8af7d9a3362add527d2e93 Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 6 Mar 2020 21:14:50 -0600 Subject: [PATCH] search: move query parsing code from tag model to post query builder. --- app/helpers/posts_helper.rb | 4 +- .../src/javascripts/autocomplete.js.erb | 4 +- app/jobs/tag_batch_change_job.rb | 8 +- app/logical/alias_and_implication_importer.rb | 4 +- app/logical/concerns/searchable.rb | 6 +- app/logical/post_query_builder.rb | 541 ++++++++++++++++- app/logical/post_sets/post.rb | 2 +- app/logical/tag_name_validator.rb | 2 +- app/models/post.rb | 10 +- app/models/saved_search.rb | 6 +- app/models/tag.rb | 553 +----------------- test/unit/post_test.rb | 4 +- test/unit/tag_test.rb | 37 +- 13 files changed, 583 insertions(+), 598 deletions(-) diff --git a/app/helpers/posts_helper.rb b/app/helpers/posts_helper.rb index f1ea2f60a..12cd1a2eb 100644 --- a/app/helpers/posts_helper.rb +++ b/app/helpers/posts_helper.rb @@ -25,7 +25,7 @@ module PostsHelper return unless post_search_counts_enabled? return unless params[:action] == "index" && params[:page].nil? && params[:tags].present? - tags = Tag.scan_query(params[:tags]).sort.join(" ") + tags = PostQueryBuilder.scan_query(params[:tags]).sort.join(" ") sig = generate_reportbooru_signature("ps-#{tags}") render "posts/partials/index/search_count", sig: sig end @@ -63,7 +63,7 @@ module PostsHelper end def show_tag_change_notice? - Tag.scan_query(params[:tags]).size == 1 && TagChangeNoticeService.get_forum_topic_id(params[:tags]) + PostQueryBuilder.scan_query(params[:tags]).size == 1 && TagChangeNoticeService.get_forum_topic_id(params[:tags]) end private diff --git a/app/javascript/src/javascripts/autocomplete.js.erb b/app/javascript/src/javascripts/autocomplete.js.erb index 9e052579b..c207da5fc 100644 --- a/app/javascript/src/javascripts/autocomplete.js.erb +++ b/app/javascript/src/javascripts/autocomplete.js.erb @@ -3,9 +3,9 @@ import CurrentUser from './current_user' let Autocomplete = {}; /* eslint-disable */ -Autocomplete.METATAGS = <%= Tag::METATAGS.to_json.html_safe %>; +Autocomplete.METATAGS = <%= PostQueryBuilder::METATAGS.to_json.html_safe %>; Autocomplete.TAG_CATEGORIES = <%= TagCategory.mapping.to_json.html_safe %>; -Autocomplete.ORDER_METATAGS = <%= Tag::ORDER_METATAGS.to_json.html_safe %>; +Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_safe %>; /* eslint-enable */ Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|"); diff --git a/app/jobs/tag_batch_change_job.rb b/app/jobs/tag_batch_change_job.rb index 542752d13..17ec1b4c9 100644 --- a/app/jobs/tag_batch_change_job.rb +++ b/app/jobs/tag_batch_change_job.rb @@ -6,8 +6,8 @@ class TagBatchChangeJob < ApplicationJob def perform(antecedent, consequent, updater, updater_ip_addr) raise Error.new("antecedent is missing") if antecedent.blank? - normalized_antecedent = TagAlias.to_aliased(::Tag.scan_tags(antecedent.mb_chars.downcase)) - normalized_consequent = TagAlias.to_aliased(::Tag.scan_tags(consequent.mb_chars.downcase)) + normalized_antecedent = TagAlias.to_aliased(PostQueryBuilder.scan_query(antecedent.mb_chars.downcase)) + normalized_consequent = TagAlias.to_aliased(PostQueryBuilder.scan_query(consequent.mb_chars.downcase)) CurrentUser.without_safe_mode do CurrentUser.scoped(updater, updater_ip_addr) do @@ -30,7 +30,7 @@ class TagBatchChangeJob < ApplicationJob end def migrate_saved_searches(normalized_antecedent, normalized_consequent) - tags = Tag.scan_tags(normalized_antecedent.join(" "), strip_metatags: true) + tags = PostQueryBuilder.scan_query(normalized_antecedent.join(" "), strip_metatags: true) # https://www.postgresql.org/docs/current/static/functions-array.html saved_searches = SavedSearch.where("string_to_array(query, ' ') @> ARRAY[?]", tags) @@ -53,7 +53,7 @@ class TagBatchChangeJob < ApplicationJob begin repl = user.blacklisted_tags.split(/\r\n|\r|\n/).map do |line| - list = Tag.scan_tags(line) + list = PostQueryBuilder.scan_query(line) if (list & query).size != query.size next line diff --git a/app/logical/alias_and_implication_importer.rb b/app/logical/alias_and_implication_importer.rb index ccf5502e0..c55979399 100644 --- a/app/logical/alias_and_implication_importer.rb +++ b/app/logical/alias_and_implication_importer.rb @@ -88,8 +88,8 @@ class AliasAndImplicationImporter all when :mass_update - all += Tag.scan_tags(token[1]) - all += Tag.scan_tags(token[2]) + all += PostQueryBuilder.scan_query(token[1]) + all += PostQueryBuilder.scan_query(token[2]) all when :change_category diff --git a/app/logical/concerns/searchable.rb b/app/logical/concerns/searchable.rb index ea40005ae..ce5265ecc 100644 --- a/app/logical/concerns/searchable.rb +++ b/app/logical/concerns/searchable.rb @@ -64,7 +64,7 @@ module Searchable def where_array_count(attr, value) relation = all qualified_column = "cardinality(#{qualified_column_for(attr)})" - parsed_range = Tag.parse_helper(value, :integer) + parsed_range = PostQueryBuilder.parse_helper(value, :integer) PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, relation) end @@ -96,7 +96,7 @@ module Searchable column = column_for_attribute(attribute) qualified_column = "#{table_name}.#{column.name}" - parsed_range = Tag.parse_helper(range, column.type) + parsed_range = PostQueryBuilder.parse_helper(range, column.type) PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, self) end @@ -252,7 +252,7 @@ module Searchable def apply_default_order(params) if params[:order] == "custom" - parse_ids = Tag.parse_helper(params[:id]) + parse_ids = PostQueryBuilder.parse_helper(params[:id]) if parse_ids[0] == :in return find_ordered(parse_ids[1]) end diff --git a/app/logical/post_query_builder.rb b/app/logical/post_query_builder.rb index be3a5f839..663a71153 100644 --- a/app/logical/post_query_builder.rb +++ b/app/logical/post_query_builder.rb @@ -1,4 +1,49 @@ class PostQueryBuilder + COUNT_METATAGS = %w[ + comment_count deleted_comment_count active_comment_count + note_count deleted_note_count active_note_count + flag_count resolved_flag_count unresolved_flag_count + child_count deleted_child_count active_child_count + pool_count deleted_pool_count active_pool_count series_pool_count collection_pool_count + appeal_count approval_count replacement_count + ] + + # allow e.g. `deleted_comments` as a synonym for `deleted_comment_count` + COUNT_METATAG_SYNONYMS = COUNT_METATAGS.map { |str| str.delete_suffix("_count").pluralize } + + METATAGS = %w[ + -user user -approver approver commenter comm noter noteupdater artcomm + -pool pool ordpool -favgroup favgroup -fav fav ordfav md5 -rating rating + -locked locked width height mpixels ratio score favcount filesize source + -source id -id date age order limit -status status tagcount parent -parent + child pixiv_id pixiv search upvote downvote filetype -filetype flagger + -flagger appealer -appealer disapproved -disapproved embedded + ] + TagCategory.short_name_list.map {|x| "#{x}tags"} + COUNT_METATAGS + COUNT_METATAG_SYNONYMS + + ORDER_METATAGS = %w[ + id id_desc + score score_asc + favcount favcount_asc + created_at created_at_asc + change change_asc + comment comment_asc + comment_bumped comment_bumped_asc + note note_asc + artcomm artcomm_asc + mpixels mpixels_asc + portrait landscape + filesize filesize_asc + tagcount tagcount_asc + rank + curated + modqueue + random + custom + ] + + COUNT_METATAGS + + COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } + + TagCategory.short_name_list.flat_map { |str| ["#{str}tags", "#{str}tags_asc"] } + attr_accessor :query_string def initialize(query_string) @@ -81,7 +126,7 @@ class PostQueryBuilder end def table_for_metatag(metatag) - if metatag.in?(Tag::COUNT_METATAGS) + if metatag.in?(COUNT_METATAGS) metatag[/(?[a-z]+)_count\z/i, :table] else nil @@ -111,7 +156,7 @@ class PostQueryBuilder def build unless query_string.is_a?(Hash) - q = Tag.parse_query(query_string) + q = PostQueryBuilder.parse_query(query_string) end relation = Post.all @@ -140,7 +185,7 @@ class PostQueryBuilder end relation = add_range_relation(q[:post_tag_count], "posts.tag_count", relation) - Tag::COUNT_METATAGS.each do |column| + COUNT_METATAGS.each do |column| relation = add_range_relation(q[column.to_sym], "posts.#{column}", relation) end @@ -582,7 +627,7 @@ class PostQueryBuilder when "filesize_asc" relation = relation.order("posts.file_size ASC") - when /\A(?#{Tag::COUNT_METATAGS.join("|")})(_(?asc|desc))?\z/i + when /\A(?#{COUNT_METATAGS.join("|")})(_(?asc|desc))?\z/i column = $~[:column] direction = $~[:direction] || "desc" relation = relation.order(column => direction, :id => direction) @@ -624,4 +669,492 @@ class PostQueryBuilder relation end + + concerning :ParseMethods do + class_methods do + def scan_query(query, strip_metatags: false) + tagstr = query.to_s.gsub(/\u3000/, " ").strip + list = tagstr.scan(/-?source:".*?"/) || [] + list += tagstr.gsub(/-?source:".*?"/, "").scan(/[^[:space:]]+/).uniq + list = list.map { |tag| tag.sub(/^[-~]/, "") } if strip_metatags + list + end + + def normalize_query(query, normalize_aliases: true, sort: true) + tags = scan_query(query.to_s) + tags = tags.map { |t| Tag.normalize_name(t) } + tags = TagAlias.to_aliased(tags) if normalize_aliases + tags = tags.sort if sort + tags = tags.uniq + tags.join(" ") + end + + def parse_query(query, options = {}) + q = {} + + q[:tag_count] = 0 + + q[:tags] = { + :related => [], + :include => [], + :exclude => [] + } + + scan_query(query).each do |token| + q[:tag_count] += 1 unless Danbooru.config.is_unlimited_tag?(token) + + if token =~ /\A(#{METATAGS.join("|")}):(.+)\z/i + g1 = $1.downcase + g2 = $2 + case g1 + when "-user" + q[:uploader_id_neg] ||= [] + user_id = User.name_to_id(g2) + q[:uploader_id_neg] << user_id unless user_id.blank? + + when "user" + user_id = User.name_to_id(g2) + q[:uploader_id] = user_id unless user_id.blank? + + when "-approver" + if g2 == "none" + q[:approver_id] = "any" + elsif g2 == "any" + q[:approver_id] = "none" + else + q[:approver_id_neg] ||= [] + user_id = User.name_to_id(g2) + q[:approver_id_neg] << user_id unless user_id.blank? + end + + when "approver" + if g2 == "none" + q[:approver_id] = "none" + elsif g2 == "any" + q[:approver_id] = "any" + else + user_id = User.name_to_id(g2) + q[:approver_id] = user_id unless user_id.blank? + end + + when "flagger" + q[:flagger_ids] ||= [] + + if g2 == "none" + q[:flagger_ids] << "none" + elsif g2 == "any" + q[:flagger_ids] << "any" + else + user_id = User.name_to_id(g2) + q[:flagger_ids] << user_id unless user_id.blank? + end + + when "-flagger" + if g2 == "none" + q[:flagger_ids] ||= [] + q[:flagger_ids] << "any" + elsif g2 == "any" + q[:flagger_ids] ||= [] + q[:flagger_ids] << "none" + else + q[:flagger_ids_neg] ||= [] + user_id = User.name_to_id(g2) + q[:flagger_ids_neg] << user_id unless user_id.blank? + end + + when "appealer" + q[:appealer_ids] ||= [] + + if g2 == "none" + q[:appealer_ids] << "none" + elsif g2 == "any" + q[:appealer_ids] << "any" + else + user_id = User.name_to_id(g2) + q[:appealer_ids] << user_id unless user_id.blank? + end + + when "-appealer" + if g2 == "none" + q[:appealer_ids] ||= [] + q[:appealer_ids] << "any" + elsif g2 == "any" + q[:appealer_ids] ||= [] + q[:appealer_ids] << "none" + else + q[:appealer_ids_neg] ||= [] + user_id = User.name_to_id(g2) + q[:appealer_ids_neg] << user_id unless user_id.blank? + end + + when "commenter", "comm" + q[:commenter_ids] ||= [] + + if g2 == "none" + q[:commenter_ids] << "none" + elsif g2 == "any" + q[:commenter_ids] << "any" + else + user_id = User.name_to_id(g2) + q[:commenter_ids] << user_id unless user_id.blank? + end + + when "noter" + q[:noter_ids] ||= [] + + if g2 == "none" + q[:noter_ids] << "none" + elsif g2 == "any" + q[:noter_ids] << "any" + else + user_id = User.name_to_id(g2) + q[:noter_ids] << user_id unless user_id.blank? + end + + when "noteupdater" + q[:note_updater_ids] ||= [] + user_id = User.name_to_id(g2) + q[:note_updater_ids] << user_id unless user_id.blank? + + when "artcomm" + q[:artcomm_ids] ||= [] + user_id = User.name_to_id(g2) + q[:artcomm_ids] << user_id unless user_id.blank? + + when "disapproved" + q[:disapproved] ||= [] + q[:disapproved] << g2 + + when "-disapproved" + q[:disapproved_neg] ||= [] + q[:disapproved_neg] << g2 + + when "-pool" + q[:pool_neg] ||= [] + q[:pool_neg] << g2 + + when "pool" + q[:pool] ||= [] + q[:pool] << g2 + + when "ordpool" + q[:ordpool] = g2 + + when "-favgroup" + favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user) + raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user) + + q[:favgroups_neg] ||= [] + q[:favgroups_neg] << favgroup + + when "favgroup" + favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user) + raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user) + + q[:favgroups] ||= [] + q[:favgroups] << favgroup + + when "-fav" + favuser = User.find_by_name(g2) + + if favuser.hide_favorites? + raise User::PrivilegeError.new + end + + q[:tags][:exclude] << "fav:#{User.name_to_id(g2)}" + + when "fav" + favuser = User.find_by_name(g2) + + if favuser.hide_favorites? + raise User::PrivilegeError.new + end + + q[:tags][:related] << "fav:#{User.name_to_id(g2)}" + + when "ordfav" + user_id = User.name_to_id(g2) + favuser = User.find(user_id) + + if favuser.hide_favorites? + raise User::PrivilegeError.new + end + + q[:tags][:related] << "fav:#{user_id}" + q[:ordfav] = user_id + + when "search" + q[:saved_searches] ||= [] + q[:saved_searches] << g2 + + when "md5" + q[:md5] = g2.downcase.split(/,/) + + when "-rating" + q[:rating_negated] = g2.downcase + + when "rating" + q[:rating] = g2.downcase + + when "-locked" + q[:locked_negated] = g2.downcase + + when "locked" + q[:locked] = g2.downcase + + when "id" + q[:post_id] = parse_helper(g2) + + when "-id" + q[:post_id_negated] = g2.to_i + + when "width" + q[:width] = parse_helper(g2) + + when "height" + q[:height] = parse_helper(g2) + + when "mpixels" + q[:mpixels] = parse_helper_fudged(g2, :float) + + when "ratio" + q[:ratio] = parse_helper(g2, :ratio) + + when "score" + q[:score] = parse_helper(g2) + + when "favcount" + q[:fav_count] = parse_helper(g2) + + when "filesize" + q[:filesize] = parse_helper_fudged(g2, :filesize) + + when "source" + q[:source] = g2.gsub(/\A"(.*)"\Z/, '\1') + + when "-source" + q[:source_neg] = g2.gsub(/\A"(.*)"\Z/, '\1') + + when "date" + q[:date] = parse_helper(g2, :date) + + when "age" + q[:age] = reverse_parse_helper(parse_helper(g2, :age)) + + when "tagcount" + q[:post_tag_count] = parse_helper(g2) + + when /(#{TagCategory.short_name_regex})tags/ + q["#{TagCategory.short_name_mapping[$1]}_tag_count".to_sym] = parse_helper(g2) + + when "parent" + q[:parent] = g2.downcase + + when "-parent" + if g2.downcase == "none" + q[:parent] = "any" + elsif g2.downcase == "any" + q[:parent] = "none" + else + q[:parent_neg_ids] ||= [] + q[:parent_neg_ids] << g2.downcase + end + + when "child" + q[:child] = g2.downcase + + when "order" + g2 = g2.downcase + + order, suffix, _tail = g2.partition(/_(asc|desc)\z/i) + if order.in?(COUNT_METATAG_SYNONYMS) + g2 = order.singularize + "_count" + suffix + end + + q[:order] = g2 + + when "limit" + # Do nothing. The controller takes care of it. + + when "-status" + q[:status_neg] = g2.downcase + + when "status" + q[:status] = g2.downcase + + when "embedded" + q[:embedded] = g2.downcase + + when "filetype" + q[:filetype] = g2.downcase + + when "-filetype" + q[:filetype_neg] = g2.downcase + + when "pixiv_id", "pixiv" + if g2.downcase == "any" || g2.downcase == "none" + q[:pixiv_id] = g2.downcase + else + q[:pixiv_id] = parse_helper(g2) + end + + when "upvote" + if CurrentUser.user.is_admin? + q[:upvote] = User.find_by_name(g2) + elsif CurrentUser.user.is_voter? + q[:upvote] = CurrentUser.user + end + + when "downvote" + if CurrentUser.user.is_admin? + q[:downvote] = User.find_by_name(g2) + elsif CurrentUser.user.is_voter? + q[:downvote] = CurrentUser.user + end + + when *COUNT_METATAGS + q[g1.to_sym] = parse_helper(g2) + + when *COUNT_METATAG_SYNONYMS + g1 = "#{g1.singularize}_count" + q[g1.to_sym] = parse_helper(g2) + + end + + else + parse_tag(token, q[:tags]) + end + end + + q[:tags][:exclude] = TagAlias.to_aliased(q[:tags][:exclude]) + q[:tags][:include] = TagAlias.to_aliased(q[:tags][:include]) + q[:tags][:related] = TagAlias.to_aliased(q[:tags][:related]) + + return q + end + + def parse_tag(tag, output) + if tag[0] == "-" && tag.size > 1 + output[:exclude] << tag[1..-1].mb_chars.downcase + + elsif tag[0] == "~" && tag.size > 1 + output[:include] << tag[1..-1].mb_chars.downcase + + elsif tag =~ /\*/ + matches = Tag.name_matches(tag).select("name").limit(Danbooru.config.tag_query_limit).order("post_count DESC").map(&:name) + matches = ["~no_matches~"] if matches.empty? + output[:include] += matches + + else + output[:related] << tag.mb_chars.downcase + end + end + + def parse_cast(object, type) + case type + when :integer + object.to_i + + when :float + object.to_f + + when :date, :datetime + Time.zone.parse(object) rescue nil + + when :age + DurationParser.parse(object).ago + + when :ratio + object =~ /\A(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)\Z/i + + if $1 && $2.to_f != 0.0 + ($1.to_f / $2.to_f).round(2) + else + object.to_f.round(2) + end + + when :filesize + object =~ /\A(\d+(?:\.\d*)?|\d*\.\d+)([kKmM]?)[bB]?\Z/ + + size = $1.to_f + unit = $2 + + conversion_factor = case unit + when /m/i + 1024 * 1024 + when /k/i + 1024 + else + 1 + end + + (size * conversion_factor).to_i + end + end + + def parse_helper(range, type = :integer) + # "1", "0.5", "5.", ".5": + # (-?(\d+(\.\d*)?|\d*\.\d+)) + case range + when /\A(.+?)\.\.(.+)/ + return [:between, parse_cast($1, type), parse_cast($2, type)] + + when /\A<=(.+)/, /\A\.\.(.+)/ + return [:lte, parse_cast($1, type)] + + when /\A<(.+)/ + return [:lt, parse_cast($1, type)] + + when /\A>=(.+)/, /\A(.+)\.\.\Z/ + return [:gte, parse_cast($1, type)] + + when /\A>(.+)/ + return [:gt, parse_cast($1, type)] + + when /[, ]/ + return [:in, range.split(/[, ]+/).map {|x| parse_cast(x, type)}] + + else + return [:eq, parse_cast(range, type)] + + end + end + + def parse_helper_fudged(range, type) + result = parse_helper(range, type) + # Don't fudge the filesize when searching filesize:123b or filesize:123. + if result[0] == :eq && type == :filesize && range !~ /[km]b?\Z/i + result + elsif result[0] == :eq + new_min = (result[1] * 0.95).to_i + new_max = (result[1] * 1.05).to_i + [:between, new_min, new_max] + else + result + end + end + + def reverse_parse_helper(array) + case array[0] + when :between + [:between, *array[1..-1].reverse] + + when :lte + [:gte, *array[1..-1]] + + when :lt + [:gt, *array[1..-1]] + + when :gte + [:lte, *array[1..-1]] + + when :gt + [:lt, *array[1..-1]] + + else + array + end + end + end + end end diff --git a/app/logical/post_sets/post.rb b/app/logical/post_sets/post.rb index a5347d7fe..0470fba84 100644 --- a/app/logical/post_sets/post.rb +++ b/app/logical/post_sets/post.rb @@ -4,7 +4,7 @@ module PostSets attr_reader :tag_array, :page, :raw, :random, :post_count, :format def initialize(tags, page = 1, per_page = nil, raw: false, random: false, format: "html") - @tag_array = Tag.scan_query(tags) + @tag_array = PostQueryBuilder.scan_query(tags) @page = page @per_page = per_page @raw = raw.to_s.truthy? diff --git a/app/logical/tag_name_validator.rb b/app/logical/tag_name_validator.rb index 9cbcecad3..c890f1695 100644 --- a/app/logical/tag_name_validator.rb +++ b/app/logical/tag_name_validator.rb @@ -21,7 +21,7 @@ class TagNameValidator < ActiveModel::EachValidator record.errors[attribute] << "'#{value}' cannot contain non-printable characters" when /[^[:ascii:]]/ record.errors[attribute] << "'#{value}' must consist of only ASCII characters" - when /\A(#{Tag::METATAGS.join("|")}):(.+)\z/i + when /\A(#{PostQueryBuilder::METATAGS.join("|")}):(.+)\z/i record.errors[attribute] << "'#{value}' cannot begin with '#{$1}:'" when /\A(#{Tag.categories.regexp}):(.+)\z/i record.errors[attribute] << "'#{value}' cannot begin with '#{$1}:'" diff --git a/app/models/post.rb b/app/models/post.rb index 21cf08d1c..a0f0071d8 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -522,11 +522,11 @@ class Post < ApplicationRecord module TagMethods def tag_array - @tag_array ||= Tag.scan_tags(tag_string) + @tag_array ||= PostQueryBuilder.scan_query(tag_string) end def tag_array_was - @tag_array_was ||= Tag.scan_tags(tag_string_in_database.presence || tag_string_before_last_save || "") + @tag_array_was ||= PostQueryBuilder.scan_query(tag_string_in_database.presence || tag_string_before_last_save || "") end def tags @@ -590,7 +590,7 @@ class Post < ApplicationRecord # then try to merge the tag changes together. current_tags = tag_array_was new_tags = tag_array - old_tags = Tag.scan_tags(old_tag_string) + old_tags = PostQueryBuilder.scan_query(old_tag_string) kept_tags = current_tags & new_tags @removed_tags = old_tags - kept_tags @@ -627,7 +627,7 @@ class Post < ApplicationRecord end def normalize_tags - normalized_tags = Tag.scan_tags(tag_string) + normalized_tags = PostQueryBuilder.scan_query(tag_string) normalized_tags = apply_casesensitive_metatags(normalized_tags) normalized_tags = normalized_tags.map(&:downcase) normalized_tags = filter_metatags(normalized_tags) @@ -1058,7 +1058,7 @@ class Post < ApplicationRecord tags = tags.to_s tags += " rating:s" if CurrentUser.safe_mode? tags += " -status:deleted" if CurrentUser.hide_deleted_posts? && !Tag.has_metatag?(tags, "status", "-status") - tags = Tag.normalize_query(tags) + tags = PostQueryBuilder.normalize_query(tags) # Optimize some cases. these are just estimates but at these # quantities being off by a few hundred doesn't matter much diff --git a/app/models/saved_search.rb b/app/models/saved_search.rb index 22f6c6151..548201648 100644 --- a/app/models/saved_search.rb +++ b/app/models/saved_search.rb @@ -139,18 +139,18 @@ class SavedSearch < ApplicationRecord .where(user_id: user_id) .labeled(label) .pluck(:query) - .map {|x| Tag.normalize_query(x, sort: true)} + .map {|x| PostQueryBuilder.normalize_query(x, sort: true)} .sort .uniq end end def normalized_query - Tag.normalize_query(query, sort: true) + PostQueryBuilder.normalize_query(query, sort: true) end def normalize_query - self.query = Tag.normalize_query(query, sort: false) + self.query = PostQueryBuilder.normalize_query(query, sort: false) end end diff --git a/app/models/tag.rb b/app/models/tag.rb index b283e42ee..5f78f26a8 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -1,51 +1,4 @@ class Tag < ApplicationRecord - COUNT_METATAGS = %w[ - comment_count deleted_comment_count active_comment_count - note_count deleted_note_count active_note_count - flag_count resolved_flag_count unresolved_flag_count - child_count deleted_child_count active_child_count - pool_count deleted_pool_count active_pool_count series_pool_count collection_pool_count - appeal_count approval_count replacement_count - ] - - # allow e.g. `deleted_comments` as a synonym for `deleted_comment_count` - COUNT_METATAG_SYNONYMS = COUNT_METATAGS.map { |str| str.delete_suffix("_count").pluralize } - - METATAGS = %w[ - -user user -approver approver commenter comm noter noteupdater artcomm - -pool pool ordpool -favgroup favgroup -fav fav ordfav md5 -rating rating - -locked locked width height mpixels ratio score favcount filesize source - -source id -id date age order limit -status status tagcount parent -parent - child pixiv_id pixiv search upvote downvote filetype -filetype flagger - -flagger appealer -appealer disapproved -disapproved embedded - ] + TagCategory.short_name_list.map {|x| "#{x}tags"} + COUNT_METATAGS + COUNT_METATAG_SYNONYMS - - SUBQUERY_METATAGS = %w[commenter comm noter noteupdater artcomm flagger -flagger appealer -appealer] - - ORDER_METATAGS = %w[ - id id_desc - score score_asc - favcount favcount_asc - created_at created_at_asc - change change_asc - comment comment_asc - comment_bumped comment_bumped_asc - note note_asc - artcomm artcomm_asc - mpixels mpixels_asc - portrait landscape - filesize filesize_asc - tagcount tagcount_asc - rank - curated - modqueue - random - custom - ] + - COUNT_METATAGS + - COUNT_METATAG_SYNONYMS.flat_map { |str| [str, "#{str}_asc"] } + - TagCategory.short_name_list.flat_map { |str| ["#{str}tags", "#{str}tags_asc"] } - has_one :wiki_page, :foreign_key => "title", :primary_key => "name" has_one :artist, :foreign_key => "name", :primary_key => "name" has_one :antecedent_alias, -> {active}, :class_name => "TagAlias", :foreign_key => "antecedent_name", :primary_key => "name" @@ -268,169 +221,17 @@ class Tag < ApplicationRecord end module ParseMethods - def normalize(query) - query.to_s.gsub(/\u3000/, " ").strip - end - - def normalize_query(query, normalize_aliases: true, sort: true) - tags = Tag.scan_query(query.to_s) - tags = tags.map { |t| Tag.normalize_name(t) } - tags = TagAlias.to_aliased(tags) if normalize_aliases - tags = tags.sort if sort - tags = tags.uniq - tags.join(" ") - end - - def scan_query(query) - tagstr = normalize(query) - list = tagstr.scan(/-?source:".*?"/) || [] - list + tagstr.gsub(/-?source:".*?"/, "").scan(/[^[:space:]]+/).uniq - end - - def scan_tags(tags, options = {}) - tagstr = normalize(tags) - list = tagstr.scan(/source:".*?"/) || [] - list += tagstr.gsub(/source:".*?"/, "").scan(/[^[:space:]]+/).uniq - if options[:strip_metatags] - list = list.map {|x| x.sub(/^[-~]/, "")} - end - list - end - - def parse_cast(object, type) - case type - when :integer - object.to_i - - when :float - object.to_f - - when :date, :datetime - Time.zone.parse(object) rescue nil - - when :age - DurationParser.parse(object).ago - - when :ratio - object =~ /\A(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)\Z/i - - if $1 && $2.to_f != 0.0 - ($1.to_f / $2.to_f).round(2) - else - object.to_f.round(2) - end - - when :filesize - object =~ /\A(\d+(?:\.\d*)?|\d*\.\d+)([kKmM]?)[bB]?\Z/ - - size = $1.to_f - unit = $2 - - conversion_factor = case unit - when /m/i - 1024 * 1024 - when /k/i - 1024 - else - 1 - end - - (size * conversion_factor).to_i - end - end - - def parse_helper(range, type = :integer) - # "1", "0.5", "5.", ".5": - # (-?(\d+(\.\d*)?|\d*\.\d+)) - case range - when /\A(.+?)\.\.(.+)/ - return [:between, parse_cast($1, type), parse_cast($2, type)] - - when /\A<=(.+)/, /\A\.\.(.+)/ - return [:lte, parse_cast($1, type)] - - when /\A<(.+)/ - return [:lt, parse_cast($1, type)] - - when /\A>=(.+)/, /\A(.+)\.\.\Z/ - return [:gte, parse_cast($1, type)] - - when /\A>(.+)/ - return [:gt, parse_cast($1, type)] - - when /[, ]/ - return [:in, range.split(/[, ]+/).map {|x| parse_cast(x, type)}] - - else - return [:eq, parse_cast(range, type)] - - end - end - - def parse_helper_fudged(range, type) - result = parse_helper(range, type) - # Don't fudge the filesize when searching filesize:123b or filesize:123. - if result[0] == :eq && type == :filesize && range !~ /[km]b?\Z/i - result - elsif result[0] == :eq - new_min = (result[1] * 0.95).to_i - new_max = (result[1] * 1.05).to_i - [:between, new_min, new_max] - else - result - end - end - - def reverse_parse_helper(array) - case array[0] - when :between - [:between, *array[1..-1].reverse] - - when :lte - [:gte, *array[1..-1]] - - when :lt - [:gt, *array[1..-1]] - - when :gte - [:lte, *array[1..-1]] - - when :gt - [:lt, *array[1..-1]] - - else - array - end - end - - def parse_tag(tag, output) - if tag[0] == "-" && tag.size > 1 - output[:exclude] << tag[1..-1].mb_chars.downcase - - elsif tag[0] == "~" && tag.size > 1 - output[:include] << tag[1..-1].mb_chars.downcase - - elsif tag =~ /\*/ - matches = Tag.name_matches(tag).select("name").limit(Danbooru.config.tag_query_limit).order("post_count DESC").map(&:name) - matches = ["~no_matches~"] if matches.empty? - output[:include] += matches - - else - output[:related] << tag.mb_chars.downcase - end - end - # true if query is a single "simple" tag (not a metatag, negated tag, or wildcard tag). def is_simple_tag?(query) is_single_tag?(query) && !is_metatag?(query) && !is_negated_tag?(query) && !is_optional_tag?(query) && !is_wildcard_tag?(query) end def is_single_tag?(query) - scan_query(query).size == 1 + PostQueryBuilder.scan_query(query).size == 1 end def is_metatag?(tag) - has_metatag?(tag, *METATAGS) + has_metatag?(tag, *PostQueryBuilder::METATAGS) end def is_negated_tag?(tag) @@ -448,357 +249,9 @@ class Tag < ApplicationRecord def has_metatag?(tags, *metatags) return nil if tags.blank? - tags = scan_query(tags.to_str) if tags.respond_to?(:to_str) + tags = PostQueryBuilder.scan_query(tags.to_str) if tags.respond_to?(:to_str) tags.grep(/\A(?:#{metatags.map(&:to_s).join("|")}):(.+)\z/i) { $1 }.first end - - def parse_query(query, options = {}) - q = {} - - q[:tag_count] = 0 - - q[:tags] = { - :related => [], - :include => [], - :exclude => [] - } - - scan_query(query).each do |token| - q[:tag_count] += 1 unless Danbooru.config.is_unlimited_tag?(token) - - if token =~ /\A(#{METATAGS.join("|")}):(.+)\z/i - g1 = $1.downcase - g2 = $2 - case g1 - when "-user" - q[:uploader_id_neg] ||= [] - user_id = User.name_to_id(g2) - q[:uploader_id_neg] << user_id unless user_id.blank? - - when "user" - user_id = User.name_to_id(g2) - q[:uploader_id] = user_id unless user_id.blank? - - when "-approver" - if g2 == "none" - q[:approver_id] = "any" - elsif g2 == "any" - q[:approver_id] = "none" - else - q[:approver_id_neg] ||= [] - user_id = User.name_to_id(g2) - q[:approver_id_neg] << user_id unless user_id.blank? - end - - when "approver" - if g2 == "none" - q[:approver_id] = "none" - elsif g2 == "any" - q[:approver_id] = "any" - else - user_id = User.name_to_id(g2) - q[:approver_id] = user_id unless user_id.blank? - end - - when "flagger" - q[:flagger_ids] ||= [] - - if g2 == "none" - q[:flagger_ids] << "none" - elsif g2 == "any" - q[:flagger_ids] << "any" - else - user_id = User.name_to_id(g2) - q[:flagger_ids] << user_id unless user_id.blank? - end - - when "-flagger" - if g2 == "none" - q[:flagger_ids] ||= [] - q[:flagger_ids] << "any" - elsif g2 == "any" - q[:flagger_ids] ||= [] - q[:flagger_ids] << "none" - else - q[:flagger_ids_neg] ||= [] - user_id = User.name_to_id(g2) - q[:flagger_ids_neg] << user_id unless user_id.blank? - end - - when "appealer" - q[:appealer_ids] ||= [] - - if g2 == "none" - q[:appealer_ids] << "none" - elsif g2 == "any" - q[:appealer_ids] << "any" - else - user_id = User.name_to_id(g2) - q[:appealer_ids] << user_id unless user_id.blank? - end - - when "-appealer" - if g2 == "none" - q[:appealer_ids] ||= [] - q[:appealer_ids] << "any" - elsif g2 == "any" - q[:appealer_ids] ||= [] - q[:appealer_ids] << "none" - else - q[:appealer_ids_neg] ||= [] - user_id = User.name_to_id(g2) - q[:appealer_ids_neg] << user_id unless user_id.blank? - end - - when "commenter", "comm" - q[:commenter_ids] ||= [] - - if g2 == "none" - q[:commenter_ids] << "none" - elsif g2 == "any" - q[:commenter_ids] << "any" - else - user_id = User.name_to_id(g2) - q[:commenter_ids] << user_id unless user_id.blank? - end - - when "noter" - q[:noter_ids] ||= [] - - if g2 == "none" - q[:noter_ids] << "none" - elsif g2 == "any" - q[:noter_ids] << "any" - else - user_id = User.name_to_id(g2) - q[:noter_ids] << user_id unless user_id.blank? - end - - when "noteupdater" - q[:note_updater_ids] ||= [] - user_id = User.name_to_id(g2) - q[:note_updater_ids] << user_id unless user_id.blank? - - when "artcomm" - q[:artcomm_ids] ||= [] - user_id = User.name_to_id(g2) - q[:artcomm_ids] << user_id unless user_id.blank? - - when "disapproved" - q[:disapproved] ||= [] - q[:disapproved] << g2 - - when "-disapproved" - q[:disapproved_neg] ||= [] - q[:disapproved_neg] << g2 - - when "-pool" - q[:pool_neg] ||= [] - q[:pool_neg] << g2 - - when "pool" - q[:pool] ||= [] - q[:pool] << g2 - - when "ordpool" - q[:ordpool] = g2 - - when "-favgroup" - favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user) - raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user) - - q[:favgroups_neg] ||= [] - q[:favgroups_neg] << favgroup - - when "favgroup" - favgroup = FavoriteGroup.find_by_name_or_id!(g2, CurrentUser.user) - raise User::PrivilegeError unless favgroup.viewable_by?(CurrentUser.user) - - q[:favgroups] ||= [] - q[:favgroups] << favgroup - - when "-fav" - favuser = User.find_by_name(g2) - - if favuser.hide_favorites? - raise User::PrivilegeError.new - end - - q[:tags][:exclude] << "fav:#{User.name_to_id(g2)}" - - when "fav" - favuser = User.find_by_name(g2) - - if favuser.hide_favorites? - raise User::PrivilegeError.new - end - - q[:tags][:related] << "fav:#{User.name_to_id(g2)}" - - when "ordfav" - user_id = User.name_to_id(g2) - favuser = User.find(user_id) - - if favuser.hide_favorites? - raise User::PrivilegeError.new - end - - q[:tags][:related] << "fav:#{user_id}" - q[:ordfav] = user_id - - when "search" - q[:saved_searches] ||= [] - q[:saved_searches] << g2 - - when "md5" - q[:md5] = g2.downcase.split(/,/) - - when "-rating" - q[:rating_negated] = g2.downcase - - when "rating" - q[:rating] = g2.downcase - - when "-locked" - q[:locked_negated] = g2.downcase - - when "locked" - q[:locked] = g2.downcase - - when "id" - q[:post_id] = parse_helper(g2) - - when "-id" - q[:post_id_negated] = g2.to_i - - when "width" - q[:width] = parse_helper(g2) - - when "height" - q[:height] = parse_helper(g2) - - when "mpixels" - q[:mpixels] = parse_helper_fudged(g2, :float) - - when "ratio" - q[:ratio] = parse_helper(g2, :ratio) - - when "score" - q[:score] = parse_helper(g2) - - when "favcount" - q[:fav_count] = parse_helper(g2) - - when "filesize" - q[:filesize] = parse_helper_fudged(g2, :filesize) - - when "source" - q[:source] = g2.gsub(/\A"(.*)"\Z/, '\1') - - when "-source" - q[:source_neg] = g2.gsub(/\A"(.*)"\Z/, '\1') - - when "date" - q[:date] = parse_helper(g2, :date) - - when "age" - q[:age] = reverse_parse_helper(parse_helper(g2, :age)) - - when "tagcount" - q[:post_tag_count] = parse_helper(g2) - - when /(#{TagCategory.short_name_regex})tags/ - q["#{TagCategory.short_name_mapping[$1]}_tag_count".to_sym] = parse_helper(g2) - - when "parent" - q[:parent] = g2.downcase - - when "-parent" - if g2.downcase == "none" - q[:parent] = "any" - elsif g2.downcase == "any" - q[:parent] = "none" - else - q[:parent_neg_ids] ||= [] - q[:parent_neg_ids] << g2.downcase - end - - when "child" - q[:child] = g2.downcase - - when "order" - g2 = g2.downcase - - order, suffix, _tail = g2.partition(/_(asc|desc)\z/i) - if order.in?(COUNT_METATAG_SYNONYMS) - g2 = order.singularize + "_count" + suffix - end - - q[:order] = g2 - - when "limit" - # Do nothing. The controller takes care of it. - - when "-status" - q[:status_neg] = g2.downcase - - when "status" - q[:status] = g2.downcase - - when "embedded" - q[:embedded] = g2.downcase - - when "filetype" - q[:filetype] = g2.downcase - - when "-filetype" - q[:filetype_neg] = g2.downcase - - when "pixiv_id", "pixiv" - if g2.downcase == "any" || g2.downcase == "none" - q[:pixiv_id] = g2.downcase - else - q[:pixiv_id] = parse_helper(g2) - end - - when "upvote" - if CurrentUser.user.is_admin? - q[:upvote] = User.find_by_name(g2) - elsif CurrentUser.user.is_voter? - q[:upvote] = CurrentUser.user - end - - when "downvote" - if CurrentUser.user.is_admin? - q[:downvote] = User.find_by_name(g2) - elsif CurrentUser.user.is_voter? - q[:downvote] = CurrentUser.user - end - - when *COUNT_METATAGS - q[g1.to_sym] = parse_helper(g2) - - when *COUNT_METATAG_SYNONYMS - g1 = "#{g1.singularize}_count" - q[g1.to_sym] = parse_helper(g2) - - end - - else - parse_tag(token, q[:tags]) - end - end - - normalize_tags_in_query(q) - - return q - end - - def normalize_tags_in_query(query_hash) - query_hash[:tags][:exclude] = TagAlias.to_aliased(query_hash[:tags][:exclude]) - query_hash[:tags][:include] = TagAlias.to_aliased(query_hash[:tags][:include]) - query_hash[:tags][:related] = TagAlias.to_aliased(query_hash[:tags][:related]) - end end module SearchMethods diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index b64ee78ff..52d332bf9 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -1430,7 +1430,7 @@ class PostTest < ActiveSupport::TestCase # final should be , , , final_post = Post.find(post.id) - assert_equal(%w(aaa bbb ddd eee), Tag.scan_tags(final_post.tag_string).sort) + assert_equal(%w(aaa bbb ddd eee), PostQueryBuilder.scan_query(final_post.tag_string).sort) end should "merge any tag changes that were made after loading the initial set of tags part 2" do @@ -1453,7 +1453,7 @@ class PostTest < ActiveSupport::TestCase # final should be , , , final_post = Post.find(post.id) - assert_equal(%w(aaa bbb ddd eee), Tag.scan_tags(final_post.tag_string).sort) + assert_equal(%w(aaa bbb ddd eee), PostQueryBuilder.scan_query(final_post.tag_string).sort) end should "merge any parent, source, and rating changes that were made after loading the initial set" do diff --git a/test/unit/tag_test.rb b/test/unit/tag_test.rb index 83022be57..04ef63932 100644 --- a/test/unit/tag_test.rb +++ b/test/unit/tag_test.rb @@ -93,37 +93,36 @@ class TagTest < ActiveSupport::TestCase context "A tag parser" do should "scan a query" do - assert_equal(%w(aaa bbb), Tag.scan_query("aaa bbb")) - assert_equal(%w(~AAa -BBB* -bbb*), Tag.scan_query("~AAa -BBB* -bbb*")) + assert_equal(%w(aaa bbb), PostQueryBuilder.scan_query("aaa bbb")) + assert_equal(%w(~AAa -BBB* -bbb*), PostQueryBuilder.scan_query("~AAa -BBB* -bbb*")) end should "not strip out valid characters when scanning" do - assert_equal(%w(aaa bbb), Tag.scan_tags("aaa bbb")) - assert_equal(%w(favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%), Tag.scan_tags("favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%")) + assert_equal(%w(aaa bbb), PostQueryBuilder.scan_query("aaa bbb")) + assert_equal(%w(favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%), PostQueryBuilder.scan_query("favgroup:yondemasu_yo,_azazel-san. pool:ichigo_100%")) end should "cast values" do - assert_equal(2048, Tag.parse_cast("2kb", :filesize)) - assert_equal(2097152, Tag.parse_cast("2m", :filesize)) - assert_nothing_raised {Tag.parse_cast("2009-01-01", :date)} - assert_nothing_raised {Tag.parse_cast("1234", :integer)} - assert_nothing_raised {Tag.parse_cast("1234.56", :float)} + assert_equal(2048, PostQueryBuilder.parse_cast("2kb", :filesize)) + assert_equal(2097152, PostQueryBuilder.parse_cast("2m", :filesize)) + assert_nothing_raised {PostQueryBuilder.parse_cast("2009-01-01", :date)} + assert_nothing_raised {PostQueryBuilder.parse_cast("1234", :integer)} + assert_nothing_raised {PostQueryBuilder.parse_cast("1234.56", :float)} end should "parse a query" do tag1 = FactoryBot.create(:tag, :name => "abc") tag2 = FactoryBot.create(:tag, :name => "acb") - assert_equal(["abc"], Tag.parse_query("md5:abc")[:md5]) - assert_equal([:between, 1, 2], Tag.parse_query("id:1..2")[:post_id]) - assert_equal([:gte, 1], Tag.parse_query("id:1..")[:post_id]) - assert_equal([:lte, 2], Tag.parse_query("id:..2")[:post_id]) - assert_equal([:gt, 2], Tag.parse_query("id:>2")[:post_id]) - assert_equal([:lt, 3], Tag.parse_query("id:<3")[:post_id]) - assert_equal([:lt, 3], Tag.parse_query("ID:<3")[:post_id]) + assert_equal(["abc"], PostQueryBuilder.parse_query("md5:abc")[:md5]) + assert_equal([:between, 1, 2], PostQueryBuilder.parse_query("id:1..2")[:post_id]) + assert_equal([:gte, 1], PostQueryBuilder.parse_query("id:1..")[:post_id]) + assert_equal([:lte, 2], PostQueryBuilder.parse_query("id:..2")[:post_id]) + assert_equal([:gt, 2], PostQueryBuilder.parse_query("id:>2")[:post_id]) + assert_equal([:lt, 3], PostQueryBuilder.parse_query("id:<3")[:post_id]) + assert_equal([:lt, 3], PostQueryBuilder.parse_query("ID:<3")[:post_id]) - Tag.expects(:normalize_tags_in_query).returns(nil) - assert_equal(["acb"], Tag.parse_query("a*b")[:tags][:include]) + assert_equal(["acb"], PostQueryBuilder.parse_query("a*b")[:tags][:include]) end should "parse single tags correctly" do @@ -238,7 +237,7 @@ class TagTest < ActiveSupport::TestCase should_not allow_value("東方").for(:name).on(:create) should_not allow_value("FAV:blah").for(:name).on(:create) - metatags = Tag::METATAGS + TagCategory.mapping.keys + metatags = PostQueryBuilder::METATAGS + TagCategory.mapping.keys metatags.each do |metatag| should_not allow_value("#{metatag}:foo").for(:name).on(:create) end