Fix #3266: Normalize fullwidth Unicode characters in translated tags.

This commit is contained in:
evazion
2017-08-12 17:13:52 -05:00
parent ee38d82ba0
commit 9b031a9c20
3 changed files with 18 additions and 4 deletions

View File

@@ -87,7 +87,7 @@ module Sources
# Given a tag from the source site, should return an array of corresponding Danbooru tags.
def translate_tag(untranslated_tag)
translated_tags = Tag.where(name: WikiPage.active.other_names_equal([untranslated_tag]).uniq.select(:title))
translated_tags = Tag.where(name: WikiPage.active.other_names_equal(untranslated_tag).uniq.select(:title))
if translated_tags.empty?
normalized_name = TagAlias.to_aliased([Tag.normalize_name(untranslated_tag)])

View File

@@ -45,8 +45,8 @@ class WikiPage < ApplicationRecord
end
end
def other_names_equal(names)
query_sql = names.map(&:to_escaped_for_tsquery).join(" | ")
def other_names_equal(name)
query_sql = name.unicode_normalize(:nfkc).to_escaped_for_tsquery
where("other_names_index @@ to_tsquery('danbooru', E?)", query_sql)
end
@@ -55,7 +55,7 @@ class WikiPage < ApplicationRecord
subquery = WikiPage.from("unnest(string_to_array(other_names, ' ')) AS other_name").where("other_name ILIKE ?", name.to_escaped_for_sql_like)
where(id: subquery)
else
other_names_equal([name])
other_names_equal(name)
end
end