jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

wikis: normalize Unicode characters in wiki bodies.

Browse Source 
* Introduce an abstraction for normalizing attributes. Very loosely
  modeled after https://github.com/fnando/normalize_attributes.
* Normalize wiki bodies to Unicode NFC form.
* Normalize Unicode space characters in wiki bodies (strip zero width
  spaces, normalize line endings to CRLF, normalize Unicode spaces to
  ASCII spaces).
* Trim spaces from the start and end of wiki page bodies. This may cause
  wiki page diffs to show spaces being removed even when the user didn't
  explicitly remove the spaces themselves.
This commit is contained in:
evazion
2020-12-21 03:23:19 -06:00
parent 48ff7c42cd
commit efb836ac02
6 changed files with 76 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
test/unit/string_test.rb
View File

@@ -14,4 +14,32 @@ class StringTest < ActiveSupport::TestCase
assert_equal('%*%', '*\**'.to_escaped_for_sql_like)
end
end
context "String#normalize_whitespace" do
should "normalize unicode spaces" do
assert_equal("foo bar", "foo bar".normalize_whitespace)
assert_equal("foo bar", "foo\u00A0bar".normalize_whitespace)
assert_equal("foo bar", "foo\u3000bar".normalize_whitespace)
end
should "strip zero width characters" do
assert_equal("foobar", "foo\u180Ebar".normalize_whitespace)
assert_equal("foobar", "foo\u200Bbar".normalize_whitespace)
assert_equal("foobar", "foo\u200Cbar".normalize_whitespace)
assert_equal("foobar", "foo\u200Dbar".normalize_whitespace)
assert_equal("foobar", "foo\u2060bar".normalize_whitespace)
assert_equal("foobar", "foo\uFEFFbar".normalize_whitespace)
end
should "normalize line endings" do
assert_equal("foo\r\nbar", "foo\r\nbar".normalize_whitespace)
assert_equal("foo\r\nbar", "foo\nbar".normalize_whitespace)
assert_equal("foo\r\nbar", "foo\rbar".normalize_whitespace)
assert_equal("foo\r\nbar", "foo\vbar".normalize_whitespace)
assert_equal("foo\r\nbar", "foo\fbar".normalize_whitespace)
assert_equal("foo\r\nbar", "foo\u0085bar".normalize_whitespace)
assert_equal("foo\r\nbar", "foo\u2028bar".normalize_whitespace)
assert_equal("foo\r\nbar", "foo\u2029bar".normalize_whitespace)
end
end
end