autocomplete: refactor javascript to use /autocomplete endpoint.

This refactors the autocomplete Javascript to use a single dedicated
/autocomplete.json endpoint instead of a bunch of separate endpoints.

This simplifies the autocomplete Javascript by making it so that instead
of calling a different endpoint for each type of query (for users, wiki
pages, pools, artists, etc), then having to parse the results of each
call to get the data we need, we can call a single endpoint that returns
exactly what we need.

This also means we don't have to parse searches clientside in order to
autocomplete metatags. Instead we can just pass the search term to the
server and let it parse the search, which is easy to do serverside.

Finally, this makes autocomplete easier to test, and it makes it easier
to add more sophisticated autocomplete behavior, since most of the logic
lives serverside.
This commit is contained in:
evazion
2020-12-13 00:45:22 -06:00
parent 1484f8852c
commit adc1c2c2cc
8 changed files with 345 additions and 251 deletions

View File

@@ -2,15 +2,12 @@ class AutocompleteController < ApplicationController
respond_to :xml, :json
def index
@tags = Tag.names_matches_with_aliases(params[:query], params.fetch(:limit, 10).to_i)
@query = params.dig(:search, :query)
@type = params.dig(:search, :type)
@limit = params.fetch(:limit, 10).to_i
@autocomplete = AutocompleteService.new(@query, @type, current_user: CurrentUser.user, limit: @limit)
if request.variant.opensearch?
expires_in 1.hour
results = [params[:query], @tags.map(&:pretty_name)]
respond_with(results)
else
# XXX
respond_with(@tags.map(&:attributes))
end
@results = @autocomplete.autocomplete_results
respond_with(@results)
end
end

View File

@@ -3,16 +3,10 @@ import CurrentUser from './current_user'
let Autocomplete = {};
/* eslint-disable */
Autocomplete.METATAGS = <%= PostQueryBuilder::METATAGS.to_json.html_safe %>;
Autocomplete.TAG_CATEGORIES = <%= TagCategory.mapping.to_json.html_safe %>;
Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_safe %>;
Autocomplete.DISAPPROVAL_REASONS = <%= PostDisapproval::REASONS.to_json.html_safe %>;
/* eslint-enable */
Autocomplete.MISC_STATUSES = ["deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated", "appealed"];
Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|");
Autocomplete.METATAGS_REGEX = Autocomplete.METATAGS.concat(Object.keys(Autocomplete.TAG_CATEGORIES)).join("|");
Autocomplete.TERM_REGEX = new RegExp(`([-~]*)(?:(${Autocomplete.METATAGS_REGEX}):)?(\\S*)$`, "i");
Autocomplete.MAX_RESULTS = 10;
Autocomplete.initialize_all = function() {
@@ -39,20 +33,20 @@ Autocomplete.initialize_all = function() {
this.initialize_tag_autocomplete();
this.initialize_mention_autocomplete($("form div.input.dtext textarea"));
this.initialize_fields($('[data-autocomplete="tag"]'), Autocomplete.tag_source);
this.initialize_fields($('[data-autocomplete="artist"]'), Autocomplete.artist_source);
this.initialize_fields($('[data-autocomplete="pool"]'), Autocomplete.pool_source);
this.initialize_fields($('[data-autocomplete="user"]'), Autocomplete.user_source);
this.initialize_fields($('[data-autocomplete="wiki-page"]'), Autocomplete.wiki_source);
this.initialize_fields($('[data-autocomplete="favorite-group"]'), Autocomplete.favorite_group_source);
this.initialize_fields($('[data-autocomplete="saved-search-label"]'), Autocomplete.saved_search_source);
this.initialize_fields($('[data-autocomplete="tag"]'), "tag");
this.initialize_fields($('[data-autocomplete="artist"]'), "artist");
this.initialize_fields($('[data-autocomplete="pool"]'), "pool");
this.initialize_fields($('[data-autocomplete="user"]'), "user");
this.initialize_fields($('[data-autocomplete="wiki-page"]'), "wiki_page");
this.initialize_fields($('[data-autocomplete="favorite-group"]'), "favorite_group");
this.initialize_fields($('[data-autocomplete="saved-search-label"]'), "saved_search_label");
}
}
Autocomplete.initialize_fields = function($fields, autocomplete) {
Autocomplete.initialize_fields = function($fields, type) {
$fields.autocomplete({
source: async function(request, respond) {
let results = await autocomplete(request.term);
let results = await Autocomplete.autocomplete_source(request.term, type);
respond(results);
},
});
@@ -84,7 +78,7 @@ Autocomplete.initialize_mention_autocomplete = function($fields) {
}
if (name) {
let results = await Autocomplete.user_source(name, "@");
let results = await Autocomplete.autocomplete_source(name, "mention");
resp(results);
}
}
@@ -106,76 +100,18 @@ Autocomplete.initialize_tag_autocomplete = function() {
return false;
},
source: async function(req, resp) {
var query = Autocomplete.parse_query(req.term, this.element.get(0).selectionStart);
var metatag = query.metatag;
var term = query.term;
var results = [];
switch (metatag) {
case "order":
case "status":
case "rating":
case "locked":
case "child":
case "parent":
case "filetype":
case "disapproved":
case "embedded":
results = Autocomplete.static_metatag_source(term, metatag);
break;
case "user":
case "approver":
case "commenter":
case "comm":
case "noter":
case "noteupdater":
case "commentaryupdater":
case "artcomm":
case "fav":
case "ordfav":
case "appealer":
case "flagger":
case "upvote":
case "downvote":
results = await Autocomplete.user_source(term, metatag + ":");
break;
case "pool":
case "ordpool":
results = await Autocomplete.pool_source(term, metatag + ":");
break;
case "favgroup":
case "ordfavgroup":
results = await Autocomplete.favorite_group_source(term, metatag + ":", CurrentUser.data("id"));
break;
case "search":
results = await Autocomplete.saved_search_source(term, metatag + ":");
break;
case "tag":
results = await Autocomplete.tag_source(term);
break;
default:
results = [];
break;
}
let term = Autocomplete.current_term(this.element);
let results = await Autocomplete.autocomplete_source(term, "tag_query");
resp(results);
}
});
}
Autocomplete.parse_query = function(text, caret) {
let before_caret_text = text.substring(0, caret);
let match = before_caret_text.match(Autocomplete.TERM_REGEX);
let operator = match[1];
let metatag = match[2] ? match[2].toLowerCase() : "tag";
let term = match[3];
if (metatag in Autocomplete.TAG_CATEGORIES) {
metatag = "tag";
}
return { operator: operator, metatag: metatag, term: term };
Autocomplete.current_term = function($input) {
let query = $input.get(0).value;
let caret = $input.get(0).selectionStart;
let match = query.substring(0, caret).match(/\S*/);
return match[0];
};
// Update the input field with the item currently focused in the
@@ -264,166 +200,12 @@ Autocomplete.render_item = function(list, item) {
return $list_item.appendTo(list);
};
Autocomplete.static_metatags = {
order: Autocomplete.ORDER_METATAGS,
status: ["any"].concat(Autocomplete.MISC_STATUSES),
rating: [
"safe", "questionable", "explicit"
],
locked: [
"rating", "note", "status"
],
embedded: [
"true", "false"
],
child: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
parent: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
filetype: [
"jpg", "png", "gif", "swf", "zip", "webm", "mp4"
],
commentary: [
"true", "false", "translated", "untranslated"
],
disapproved: Autocomplete.DISAPPROVAL_REASONS
}
Autocomplete.static_metatag_source = function(term, metatag) {
var sub_metatags = this.static_metatags[metatag];
var matches = sub_metatags.filter(sub_metatag => sub_metatag.startsWith(term.toLowerCase()));
matches = matches.map(sub_metatag => `${metatag}:${sub_metatag}`).sort().slice(0, Autocomplete.MAX_RESULTS);
return matches;
}
Autocomplete.tag_source = async function(term) {
if (term === "") {
return [];
}
let tags = await $.getJSON("/tags/autocomplete", {
"search[name_matches]": term,
"limit": Autocomplete.MAX_RESULTS,
"expiry": 7
});
return tags.map(function(tag) {
return {
type: "tag",
label: tag.name.replace(/_/g, " "),
antecedent: tag.antecedent_name,
value: tag.name,
category: tag.category,
source: tag.source,
weight: tag.weight,
post_count: tag.post_count
};
});
}
Autocomplete.artist_source = async function(term) {
let artists = await $.getJSON("/artists", {
"search[name_like]": term.trim().replace(/\s+/g, "_") + "*",
"search[is_deleted]": false,
"search[order]": "post_count",
"limit": Autocomplete.MAX_RESULTS,
"expiry": 7
});
return artists.map(function(artist) {
return {
type: "tag",
label: artist.name.replace(/_/g, " "),
value: artist.name,
category: Autocomplete.TAG_CATEGORIES.artist,
};
});
};
Autocomplete.wiki_source = async function(term) {
let wiki_pages = await $.getJSON("/wiki_pages", {
"search[title_normalize]": term + "*",
"search[hide_deleted]": "Yes",
"search[order]": "post_count",
"limit": Autocomplete.MAX_RESULTS,
"expiry": 7
});
return wiki_pages.map(function(wiki_page) {
return {
type: "tag",
label: wiki_page.title.replace(/_/g, " "),
value: wiki_page.title,
category: wiki_page.category_name
};
});
};
Autocomplete.user_source = async function(term, prefix = "") {
let users = await $.getJSON("/users", {
"search[order]": "post_upload_count",
"search[current_user_first]": "true",
"search[name_matches]": term + "*",
Autocomplete.autocomplete_source = function(query, type) {
return $.getJSON("/autocomplete", {
"search[query]": query,
"search[type]": type,
"limit": Autocomplete.MAX_RESULTS
});
return users.map(function(user) {
return {
type: "user",
label: user.name.replace(/_/g, " "),
value: prefix + user.name,
level: user.level_string
};
});
};
Autocomplete.pool_source = async function(term, prefix = "") {
let pools = await $.getJSON("/pools", {
"search[name_matches]": term,
"search[is_deleted]": false,
"search[order]": "post_count",
"limit": Autocomplete.MAX_RESULTS
});
return pools.map(function(pool) {
return {
type: "pool",
label: pool.name.replace(/_/g, " "),
value: prefix + pool.name,
post_count: pool.post_count,
category: pool.category
};
});
};
Autocomplete.favorite_group_source = async function(term, prefix = "", creator_id = null) {
let favgroups = await $.getJSON("/favorite_groups", {
"search[creator_id]": creator_id,
"search[name_matches]": term,
"limit": Autocomplete.MAX_RESULTS
});
return favgroups.map(function(favgroup) {
return {
label: favgroup.name.replace(/_/g, " "),
value: prefix + favgroup.name,
post_count: favgroup.post_count
};
});
};
Autocomplete.saved_search_source = async function(term, prefix = "") {
let labels = await $.getJSON("/saved_searches/labels", {
"search[label]": term + "*",
"limit": Autocomplete.MAX_RESULTS
});
return labels.map(function(label) {
return {
label: label.replace(/_/g, " "),
value: prefix + label,
};
});
}
$(document).ready(function() {

View File

@@ -0,0 +1,165 @@
class AutocompleteService
POST_STATUSES = %w[active deleted pending flagged appealed banned modqueue unmoderated]
STATIC_METATAGS = {
status: %w[any] + POST_STATUSES,
child: %w[any none] + POST_STATUSES,
parent: %w[any none] + POST_STATUSES,
rating: %w[safe questionable explicit],
locked: %w[rating note status],
embedded: %w[true false],
filetype: %w[jpg png gif swf zip webm mp4],
commentary: %w[true false translated untranslated],
disapproved: PostDisapproval::REASONS,
order: PostQueryBuilder::ORDER_METATAGS
}
attr_reader :query, :type, :limit, :current_user
def initialize(query, type, current_user: User.anonymous, limit: 10)
@query = query.to_s
@type = type.to_sym
@current_user = current_user
@limit = limit
end
def autocomplete_results
case type
when :tag_query
autocomplete_tag_query(query)
when :tag
autocomplete_tag(query)
when :artist
autocomplete_artist(query)
when :wiki_page
autocomplete_wiki_page(query)
when :user
autocomplete_user(query)
when :mention
autocomplete_mention(query)
when :pool
autocomplete_pool(query)
when :favorite_group
autocomplete_favorite_group(query)
when :saved_search_label
autocomplete_saved_search_label(query)
when :opensearch
autocomplete_opensearch(query)
else
[]
end
end
def autocomplete_tag_query(string)
term = PostQueryBuilder.new(string).terms.first
return [] if term.nil?
case term.type
when :tag
autocomplete_tag(term.name)
when :metatag
autocomplete_metatag(term.name, term.value)
end
end
def autocomplete_tag(string)
tags = Tag.names_matches_with_aliases(string, limit)
tags.map do |tag|
{ type: "tag", label: tag.name.tr("_", " "), value: tag.name, antecedent: tag.antecedent_name, category: tag.category, post_count: tag.post_count, source: nil, weight: nil }
end
end
def autocomplete_metatag(metatag, value)
results = case metatag.to_sym
when :user, :approver, :commenter, :comm, :noter, :noteupdater, :commentaryupdater,
:artcomm, :fav, :ordfav, :appealer, :flagger, :upvote, :downvote
autocomplete_user(value)
when :pool, :ordpool
autocomplete_pool(value)
when :favgroup, :ordfavgroup
autocomplete_favorite_group(value)
when :search
autocomplete_saved_search_label(value)
when *STATIC_METATAGS.keys
autocomplete_static_metatag(metatag, value)
end
results.map do |result|
{ **result, value: metatag + ":" + result[:value] }
end
end
def autocomplete_static_metatag(metatag, value)
values = STATIC_METATAGS[metatag.to_sym]
results = values.select { |v| v.starts_with?(value) }.sort.take(limit)
results.map do |v|
{ label: metatag + ":" + v, value: v }
end
end
def autocomplete_pool(string)
string = "*" + string + "*" unless string.include?("*")
pools = Pool.undeleted.name_matches(string).search(order: "post_count").limit(limit)
pools.map do |pool|
{ type: "pool", label: pool.pretty_name, value: pool.name, post_count: pool.post_count, category: pool.category }
end
end
def autocomplete_favorite_group(string)
string = "*" + string + "*" unless string.include?("*")
favgroups = FavoriteGroup.visible(current_user).where(creator: current_user).name_matches(string).search(order: "post_count").limit(limit)
favgroups.map do |favgroup|
{ label: favgroup.pretty_name, value: favgroup.name, post_count: favgroup.post_count }
end
end
def autocomplete_saved_search_label(string)
labels = SavedSearch.search_labels(current_user.id, label: string).take(limit)
labels.map do |label|
{ label: label.tr("_", " "), value: label }
end
end
def autocomplete_artist(string)
string = string + "*" unless string.include?("*")
artists = Artist.undeleted.name_matches(string).search(order: "post_count").limit(limit)
artists.map do |artist|
{ type: "tag", label: artist.pretty_name, value: artist.name, category: Tag.categories.artist }
end
end
def autocomplete_wiki_page(string)
string = string + "*" unless string.include?("*")
wiki_pages = WikiPage.undeleted.title_matches(string).search(order: "post_count").limit(limit)
wiki_pages.map do |wiki_page|
{ type: "tag", label: wiki_page.pretty_title, value: wiki_page.title, category: wiki_page.tag&.category }
end
end
def autocomplete_user(string)
string = string + "*" unless string.include?("*")
users = User.search(name_matches: string, current_user_first: true, order: "post_upload_count").limit(limit)
users.map do |user|
{ type: "user", label: user.pretty_name, value: user.name, level: user.level_string }
end
end
def autocomplete_mention(string)
autocomplete_user(string).map do |result|
{ **result, value: "@" + result[:value] }
end
end
def autocomplete_opensearch(string)
results = autocomplete_tag(string).map { |result| result[:value] }
[query, results]
end
end

View File

@@ -203,6 +203,10 @@ class Artist < ApplicationRecord
end
module SearchMethods
def name_matches(query)
where_like(:name, normalize_name(query))
end
def any_other_name_matches(regex)
where(id: Artist.from("unnest(other_names) AS other_name").where_regex("other_name", regex))
end

View File

@@ -33,6 +33,10 @@ class WikiPage < ApplicationRecord
where(title: normalize_title(title))
end
def title_matches(title)
where_like(:title, normalize_title(title))
end
def other_names_include(name)
name = normalize_other_name(name)
subquery = WikiPage.from("unnest(other_names) AS other_name").where_iequals("other_name", name)

View File

@@ -4,5 +4,5 @@
<Description><%= Danbooru.config.app_name %> search</Description>
<Image height="16" width="16" type="image/x-icon"><%= root_url %>favicon.ico</Image>
<Url type="text/html" template="<%= posts_url %>?tags={searchTerms}&amp;utm_source=opensearch"/>
<Url type="application/x-suggestions+json" template="<%= autocomplete_index_url(format: :json) %>?query={searchTerms}&amp;type=tags&amp;variant=opensearch"/>
<Url type="application/x-suggestions+json" template="<%= autocomplete_index_url(format: :json) %>?search[query]={searchTerms}&amp;type=tags&amp;search[type]=opensearch"/>
</OpenSearchDescription>

View File

@@ -1,6 +1,17 @@
require "test_helper"
class AutocompleteControllerTest < ActionDispatch::IntegrationTest
def autocomplete(query, type)
get autocomplete_index_path(search: { query: query, type: type }), as: :json
assert_response :success
response.parsed_body.map { |result| result["value"] }
end
def assert_autocomplete_equals(expected_value, query, type)
assert_equal(expected_value, autocomplete(query, type))
end
context "Autocomplete controller" do
context "index action" do
setup do
@@ -8,9 +19,20 @@ class AutocompleteControllerTest < ActionDispatch::IntegrationTest
end
should "work for opensearch queries" do
get autocomplete_index_path(query: "azur", variant: "opensearch"), as: :json
get autocomplete_index_path(search: { query: "azur", type: "opensearch" }), as: :json
assert_response :success
assert_equal(["azur", ["azur lane"]], response.parsed_body)
assert_equal(["azur", ["azur_lane"]], response.parsed_body)
end
should "work for tag queries" do
assert_autocomplete_equals(["azur_lane"], "azur", "tag_query")
assert_autocomplete_equals(["azur_lane"], "-azur", "tag_query")
assert_autocomplete_equals(["azur_lane"], "~azur", "tag_query")
assert_autocomplete_equals(["azur_lane"], "AZUR", "tag_query")
assert_autocomplete_equals(["rating:safe"], "rating:s", "tag_query")
assert_autocomplete_equals(["rating:safe"], "-rating:s", "tag_query")
end
end
end

View File

@@ -0,0 +1,120 @@
require 'test_helper'
class AutocompleteServiceTest < ActiveSupport::TestCase
def autocomplete(query, type, **options)
results = AutocompleteService.new(query, type, **options).autocomplete_results
results.map { |r| r[:value] }
end
def assert_autocomplete_includes(expected_value, query, type, **options)
assert_includes(autocomplete(query, type, **options), expected_value)
end
def assert_autocomplete_equals(expected_value, query, type, **options)
assert_equal(expected_value, autocomplete(query, type, **options))
end
context "#autocomplete method" do
should "autocomplete artists" do
create(:artist, name: "bkub")
assert_autocomplete_includes("bkub", "bk", :artist)
end
should "autocomplete wiki pages" do
create(:wiki_page, title: "help:home")
assert_autocomplete_includes("help:home", "help", :wiki_page)
end
should "autocomplete users" do
@user = create(:user, name: "fumimi")
as(@user) do
assert_autocomplete_includes("fumimi", "fu", :user)
assert_autocomplete_includes("@fumimi", "fu", :mention)
assert_autocomplete_includes("user:fumimi", "user:fu", :tag_query)
end
end
should "autocomplete pools" do
as(create(:user)) do
create(:pool, name: "Disgustingly_Adorable")
end
assert_autocomplete_includes("Disgustingly_Adorable", "disgust", :pool)
assert_autocomplete_includes("pool:Disgustingly_Adorable", "pool:disgust", :tag_query)
assert_autocomplete_includes("pool:Disgustingly_Adorable", "-pool:disgust", :tag_query)
end
should "autocomplete favorite groups" do
user = create(:user)
create(:favorite_group, name: "Stuff", creator: user)
assert_autocomplete_equals(["Stuff"], "stu", :favorite_group, current_user: user)
assert_autocomplete_equals([], "stu", :favorite_group, current_user: User.anonymous)
assert_autocomplete_equals(["favgroup:Stuff"], "favgroup:stu", :tag_query, current_user: user)
assert_autocomplete_equals([], "favgroup:stu", :tag_query, current_user: User.anonymous)
end
should "autocomplete saved search labels" do
user = create(:user)
create(:saved_search, query: "bkub", labels: ["artists"], user: user)
assert_autocomplete_equals(["artists"], "art", :saved_search_label, current_user: user)
assert_autocomplete_equals(["search:artists"], "search:art", :tag_query, current_user: user)
end
should "autocomplete single tags" do
create(:tag, name: "touhou")
assert_autocomplete_includes("touhou", "tou", :tag)
end
context "for a tag search" do
should "autocomplete tags" do
create(:tag, name: "touhou")
assert_autocomplete_includes("touhou", "tou", :tag_query)
assert_autocomplete_includes("touhou", "TOU", :tag_query)
assert_autocomplete_includes("touhou", "-tou", :tag_query)
assert_autocomplete_includes("touhou", "~tou", :tag_query)
end
should "autocomplete static metatags" do
assert_autocomplete_equals(["status:active"], "status:act", :tag_query)
assert_autocomplete_equals(["parent:active"], "parent:act", :tag_query)
assert_autocomplete_equals(["child:active"], "child:act", :tag_query)
assert_autocomplete_equals(["rating:safe"], "rating:s", :tag_query)
assert_autocomplete_equals(["rating:questionable"], "rating:q", :tag_query)
assert_autocomplete_equals(["rating:explicit"], "rating:e", :tag_query)
assert_autocomplete_equals(["locked:rating"], "locked:r", :tag_query)
assert_autocomplete_equals(["locked:status"], "locked:s", :tag_query)
assert_autocomplete_equals(["locked:note"], "locked:n", :tag_query)
assert_autocomplete_equals(["embedded:true"], "embedded:t", :tag_query)
assert_autocomplete_equals(["embedded:false"], "embedded:f", :tag_query)
assert_autocomplete_equals(["filetype:jpg"], "filetype:j", :tag_query)
assert_autocomplete_equals(["filetype:png"], "filetype:p", :tag_query)
assert_autocomplete_equals(["filetype:gif"], "filetype:g", :tag_query)
assert_autocomplete_equals(["filetype:swf"], "filetype:s", :tag_query)
assert_autocomplete_equals(["filetype:zip"], "filetype:z", :tag_query)
assert_autocomplete_equals(["filetype:webm"], "filetype:w", :tag_query)
assert_autocomplete_equals(["filetype:mp4"], "filetype:m", :tag_query)
assert_autocomplete_equals(["commentary:true"], "commentary:tru", :tag_query)
assert_autocomplete_equals(["commentary:false"], "commentary:fal", :tag_query)
assert_autocomplete_equals(["commentary:translated"], "commentary:trans", :tag_query)
assert_autocomplete_equals(["commentary:untranslated"], "commentary:untrans", :tag_query)
assert_autocomplete_equals(["disapproved:breaks_rules"], "disapproved:break", :tag_query)
assert_autocomplete_equals(["disapproved:poor_quality"], "disapproved:poor", :tag_query)
assert_autocomplete_equals(["disapproved:disinterest"], "disapproved:dis", :tag_query)
assert_autocomplete_equals(["order:score", "order:score_asc"], "order:sco", :tag_query)
end
end
end
end