saved searches: fix normalization of labels.

Fix saved searches to remove additional invalid characters from labels:

* Remove repeated spaces or underscores.
* Remove leading and trailing spaces or underscores.
* Normalize Unicode characters to NFC form.

Also add a fix script to renormalize labels in old saved searches. A few
problems with existing searches:

* Some saved searches somehow had labels containing NULL elements.
* Some had leading or trailing underscores.
* Some had repeated underscores.
* Some had non-English characters in uppercase.

app/models/saved_search.rb

@@ -5,8 +5,9 @@ class SavedSearch < ApplicationRecord
   attr_reader :disable_labels
   belongs_to :user
 
-  before_validation :normalize_query
+  normalize :query, :normalize_query
-  before_validation :normalize_labels
+  normalize :labels, :normalize_labels
+
   validates :query, presence: true
   validate :validate_count, on: :create
 
@@ -55,12 +56,13 @@ class SavedSearch < ApplicationRecord
 
   concerning :Labels do
     class_methods do
+      def normalize_labels(labels)
+        # XXX should sort and uniq, but will break some use cases.
+        labels.map { |label| normalize_label(label) }.reject(&:blank?)
+      end
+
       def normalize_label(label)
-        label
+        label.to_s.unicode_normalize(:nfc).normalize_whitespace.downcase.gsub(/[[:space:]]+/, "_").squeeze("_").gsub(/\A_|_\z/, "")
-          .to_s
-          .strip
-          .downcase
-          .gsub(/[[:space:]]/, "_")
       end
 
       def all_labels
@@ -79,10 +81,6 @@ class SavedSearch < ApplicationRecord
       end
     end
 
-    def normalize_labels
-      self.labels = labels.map {|x| SavedSearch.normalize_label(x)}.reject(&:blank?)
-    end
-
     def label_string
       labels.join(" ")
     end
@@ -90,11 +88,6 @@ class SavedSearch < ApplicationRecord
     def label_string=(val)
       self.labels = val.to_s.split(/[[:space:]]+/)
     end
-
-    def labels=(labels)
-      labels = labels.map { |label| SavedSearch.normalize_label(label) }
-      super(labels)
-    end
   end
 
   concerning :Search do
@@ -136,6 +129,10 @@ class SavedSearch < ApplicationRecord
 
   concerning :Queries do
     class_methods do
+      def normalize_query(query)
+        PostQueryBuilder.new(query.to_s).normalized_query(sort: false).to_s
+      end
+
       def queries_for(user_id, label: nil, options: {})
         searches = SavedSearch.where(user_id: user_id)
         searches = searches.labeled(label) if label.present?
@@ -156,10 +153,6 @@ class SavedSearch < ApplicationRecord
       PostQueryBuilder.new(query).normalized_query.to_s
     end
 
-    def normalize_query
-      self.query = PostQueryBuilder.new(query).normalized_query(sort: false).to_s
-    end
-
     def rewrite_query(old_name, new_name)
       self.query.gsub!(/(?:\A| )([-~])?#{Regexp.escape(old_name)}(?: |\z)/i) { " #{$1}#{new_name} " }
       self.query.strip!

script/fixes/070_renormalize_labels_in_saved_searches.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require_relative "../../config/environment"
+
+SavedSearch.transaction do
+  SavedSearch.find_each do |ss|
+    old_labels = ss.labels
+    ss.labels = old_labels
+
+    if ss.changed?
+      p [ss.id, ss.changes]
+      ss.save!
+    end
+  end
+end

test/unit/saved_search_test.rb

@@ -110,6 +110,25 @@ class SavedSearchTest < ActiveSupport::TestCase
     end
   end
 
+  context "The #normalize_labels method" do
+    subject { build(:saved_search) }
+
+    should normalize_attribute(:labels).from(["FOO"]).to(["foo"])
+    should normalize_attribute(:labels).from(["   foo"]).to(["foo"])
+    should normalize_attribute(:labels).from(["foo   "]).to(["foo"])
+    should normalize_attribute(:labels).from(["___foo"]).to(["foo"])
+    should normalize_attribute(:labels).from(["foo___"]).to(["foo"])
+    should normalize_attribute(:labels).from(["foo\n"]).to(["foo"])
+    should normalize_attribute(:labels).from(["foo bar"]).to(["foo_bar"])
+    should normalize_attribute(:labels).from(["foo   bar"]).to(["foo_bar"])
+    should normalize_attribute(:labels).from(["foo___bar"]).to(["foo_bar"])
+    should normalize_attribute(:labels).from([" _Foo Bar_ "]).to(["foo_bar"])
+    should normalize_attribute(:labels).from(["Я"]).to(["я"])
+    should normalize_attribute(:labels).from(["foo 1", "bar 2"]).to(["foo_1", "bar_2"])
+    should normalize_attribute(:labels).from(["foo", nil, "", " ", "bar"]).to(["foo", "bar"])
+    should normalize_attribute(:labels).from([nil, "", " "]).to([])
+  end
+
   context "Populating a saved search" do
     setup do
       @saved_search = create(:saved_search, query: "bkub", user: @user)