Fresh start

1 files changed, 184 insertions, 0 deletions

diff --git a/jni/ruby/test/test_unicode_normalize.rb b/jni/ruby/test/test_unicode_normalize.rb
new file mode 100644
index 0000000..fdc9d3c
--- /dev/null
+++ b/jni/ruby/test/test_unicode_normalize.rb
@@ -0,0 +1,184 @@
+# coding: utf-8
+
+# Copyright Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)
+
+require 'test/unit'
+require 'unicode_normalize/normalize'
+
+class TestUnicodeNormalize < Test::Unit::TestCase
+
+  UNICODE_VERSION = UnicodeNormalize::UNICODE_VERSION
+
+  NormTest = Struct.new :source, :NFC, :NFD, :NFKC, :NFKD, :line
+
+  def read_tests
+    IO.readlines(File.expand_path("../enc/unicode/data/#{UNICODE_VERSION}/NormalizationTest.txt", __dir__), encoding: 'utf-8')
+    .tap { |lines| assert_include(lines[0], "NormalizationTest-#{UNICODE_VERSION}.txt")}
+    .collect.with_index { |linedata, linenumber| [linedata, linenumber]}
+    .reject { |line| line[0] =~ /^[\#@]/ }
+    .collect do |line|
+      NormTest.new *(line[0].split(';').take(5).collect do |code_string|
+        code_string.split(/\s/).collect { |cp| cp.to_i(16) }.pack('U*')
+      end + [line[1]+1])
+    end
+  end
+
+  def to_codepoints(string)
+    string.codepoints.collect { |cp| cp.to_s(16).upcase.rjust(4, '0') }
+  end
+
+  def setup
+    @@tests ||= read_tests
+  rescue Errno::ENOENT => e
+    @@tests ||= []
+    skip e.message
+  end
+
+  def self.generate_test_normalize(target, normalization, source, prechecked)
+    define_method "test_normalize_to_#{target}_from_#{source}_with_#{normalization}" do
+      expected = actual = test = nil
+      mesg = proc {"#{to_codepoints(expected)} expected but was #{to_codepoints(actual)} on line #{test[:line]} (#{normalization})"}
+      @@tests.each do |t|
+        test = t
+        if prechecked.nil? or test[prechecked]==test[source]
+          expected = test[target]
+          actual = test[source].unicode_normalize(normalization)
+          assert_equal expected, actual, mesg
+        end
+      end
+    end
+  end
+
+#      source; NFC; NFD; NFKC; NFKD
+#    NFC
+#      :NFC ==  toNFC(:source) ==  toNFC(:NFC) ==  toNFC(:NFD)
+  generate_test_normalize :NFC, :nfc, :source, nil
+  generate_test_normalize :NFC, :nfc, :NFC, :source
+  generate_test_normalize :NFC, :nfc, :NFD, :source
+#      :NFKC ==  toNFC(:NFKC) ==  toNFC(:NFKD)
+  generate_test_normalize :NFKC, :nfc, :NFKC, nil
+  generate_test_normalize :NFKC, :nfc, :NFKD, :NFKC
+#
+#    NFD
+#      :NFD ==  toNFD(:source) ==  toNFD(:NFC) ==  toNFD(:NFD)
+  generate_test_normalize :NFD, :nfd, :source, nil
+  generate_test_normalize :NFD, :nfd, :NFC, :source
+  generate_test_normalize :NFD, :nfd, :NFD, :source
+#      :NFKD ==  toNFD(:NFKC) ==  toNFD(:NFKD)
+  generate_test_normalize :NFKD, :nfd, :NFKC, nil
+  generate_test_normalize :NFKD, :nfd, :NFKD, :NFKC
+#
+#    NFKC
+#      :NFKC == toNFKC(:source) == toNFKC(:NFC) == toNFKC(:NFD) == toNFKC(:NFKC) == toNFKC(:NFKD)
+  generate_test_normalize :NFKC, :nfkc, :source, nil
+  generate_test_normalize :NFKC, :nfkc, :NFC, :source
+  generate_test_normalize :NFKC, :nfkc, :NFD, :source
+  generate_test_normalize :NFKC, :nfkc, :NFKC, :NFC
+  generate_test_normalize :NFKC, :nfkc, :NFKD, :NFD
+#
+#    NFKD
+#      :NFKD == toNFKD(:source) == toNFKD(:NFC) == toNFKD(:NFD) == toNFKD(:NFKC) == toNFKD(:NFKD)
+  generate_test_normalize :NFKD, :nfkd, :source, nil
+  generate_test_normalize :NFKD, :nfkd, :NFC, :source
+  generate_test_normalize :NFKD, :nfkd, :NFD, :source
+  generate_test_normalize :NFKD, :nfkd, :NFKC, :NFC
+  generate_test_normalize :NFKD, :nfkd, :NFKD, :NFD
+
+  def self.generate_test_check_true(source, normalization)
+    define_method "test_check_true_#{source}_as_#{normalization}" do
+      test = nil
+      mesg = proc {"#{to_codepoints(test[source])} should check as #{normalization} but does not on line #{test[:line]}"}
+      @@tests.each do |t|
+        test = t
+        actual = test[source].unicode_normalized?(normalization)
+        assert_equal true, actual, mesg
+      end
+    end
+  end
+
+  def self.generate_test_check_false(source, compare, normalization)
+    define_method "test_check_false_#{source}_as_#{normalization}" do
+      test = nil
+      mesg = proc {"#{to_codepoints(test[source])} should not check as #{normalization} but does on line #{test[:line]}"}
+      @@tests.each do |t|
+        test = t
+        if test[source] != test[compare]
+          actual = test[source].unicode_normalized?(normalization)
+          assert_equal false, actual, mesg
+        end
+      end
+    end
+  end
+
+  generate_test_check_true :NFC, :nfc
+  generate_test_check_true :NFD, :nfd
+  generate_test_check_true :NFKC, :nfc
+  generate_test_check_true :NFKC, :nfkc
+  generate_test_check_true :NFKD, :nfd
+  generate_test_check_true :NFKD, :nfkd
+
+  generate_test_check_false :source, :NFD, :nfd
+  generate_test_check_false :NFC, :NFD, :nfd
+  generate_test_check_false :NFKC, :NFKD, :nfd
+  generate_test_check_false :source, :NFC, :nfc
+  generate_test_check_false :NFD, :NFC, :nfc
+  generate_test_check_false :NFKD, :NFKC, :nfc
+  generate_test_check_false :source, :NFKD, :nfkd
+  generate_test_check_false :NFC, :NFKD, :nfkd
+  generate_test_check_false :NFD, :NFKD, :nfkd
+  generate_test_check_false :NFKC, :NFKD, :nfkd
+  generate_test_check_false :source, :NFKC, :nfkc
+  generate_test_check_false :NFC, :NFKC, :nfkc
+  generate_test_check_false :NFD, :NFKC, :nfkc
+  generate_test_check_false :NFKD, :NFKC, :nfkc
+
+  def test_non_UTF_8
+    assert_equal "\u1E0A".encode('UTF-16BE'), "D\u0307".encode('UTF-16BE').unicode_normalize(:nfc)
+    assert_equal true, "\u1E0A".encode('UTF-16BE').unicode_normalized?(:nfc)
+    assert_equal false, "D\u0307".encode('UTF-16BE').unicode_normalized?(:nfc)
+  end
+
+  def test_singleton_with_accents
+    assert_equal "\u0136", "\u212A\u0327".unicode_normalize(:nfc)
+  end
+
+  def test_partial_jamo_compose
+    assert_equal "\uAC01", "\uAC00\u11A8".unicode_normalize(:nfc)
+  end
+
+  def test_partial_jamo_decompose
+    assert_equal "\u1100\u1161\u11A8", "\uAC00\u11A8".unicode_normalize(:nfd)
+  end
+
+  def test_hangul_plus_accents
+    assert_equal "\uAC00\u0323\u0300", "\uAC00\u0300\u0323".unicode_normalize(:nfc)
+    assert_equal "\uAC00\u0323\u0300", "\u1100\u1161\u0300\u0323".unicode_normalize(:nfc)
+    assert_equal "\u1100\u1161\u0323\u0300", "\uAC00\u0300\u0323".unicode_normalize(:nfd)
+    assert_equal "\u1100\u1161\u0323\u0300", "\u1100\u1161\u0300\u0323".unicode_normalize(:nfd)
+  end
+
+  def test_raise_exception_for_non_unicode_encoding
+    assert_raise(Encoding::CompatibilityError) { "abc".force_encoding('ISO-8859-1').unicode_normalize }
+    assert_raise(Encoding::CompatibilityError) { "abc".force_encoding('ISO-8859-1').unicode_normalize! }
+    assert_raise(Encoding::CompatibilityError) { "abc".force_encoding('ISO-8859-1').unicode_normalized? }
+  end
+
+  def test_us_ascii
+    ascii_string = 'abc'.encode('US-ASCII')
+
+    assert_equal ascii_string, ascii_string.unicode_normalize
+    assert_equal ascii_string, ascii_string.unicode_normalize(:nfd)
+    assert_equal ascii_string, ascii_string.unicode_normalize(:nfkc)
+    assert_equal ascii_string, ascii_string.unicode_normalize(:nfkd)
+
+    assert_equal ascii_string, ascii_string.unicode_normalize!
+    assert_equal ascii_string, ascii_string.unicode_normalize!(:nfd)
+    assert_equal ascii_string, ascii_string.unicode_normalize!(:nfkc)
+    assert_equal ascii_string, ascii_string.unicode_normalize!(:nfkd)
+
+    assert_equal true, ascii_string.unicode_normalized?
+    assert_equal true, ascii_string.unicode_normalized?(:nfd)
+    assert_equal true, ascii_string.unicode_normalized?(:nfkc)
+    assert_equal true, ascii_string.unicode_normalized?(:nfkd)
+  end
+end