From fcbf63e62c627deae76c1b8cb8c0876c536ed811 Mon Sep 17 00:00:00 2001
From: Jari Vetoniemi <jari.vetoniemi@indooratlas.com>
Date: Mon, 16 Mar 2020 18:49:26 +0900
Subject: Fresh start

---
 jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb | 421 +++++++++++++++++++++++++++++++
 1 file changed, 421 insertions(+)
 create mode 100644 jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb

(limited to 'jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb')

diff --git a/jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb b/jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb
new file mode 100644
index 0000000..8871b4f
--- /dev/null
+++ b/jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb
@@ -0,0 +1,421 @@
+# coding: UTF-8
+
+require 'rdoc/test_case'
+
+class TestRDocRubyLex < RDoc::TestCase
+
+  def setup
+    @TK = RDoc::RubyToken
+  end
+
+  def test_class_tokenize
+    tokens = RDoc::RubyLex.tokenize "def x() end", nil
+
+    expected = [
+      @TK::TkDEF       .new( 0, 1,  0, "def"),
+      @TK::TkSPACE     .new( 3, 1,  3, " "),
+      @TK::TkIDENTIFIER.new( 4, 1,  4, "x"),
+      @TK::TkLPAREN    .new( 5, 1,  5, "("),
+      @TK::TkRPAREN    .new( 6, 1,  6, ")"),
+      @TK::TkSPACE     .new( 7, 1,  7, " "),
+      @TK::TkEND       .new( 8, 1,  8, "end"),
+      @TK::TkNL        .new(11, 1, 11, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize___END__
+    tokens = RDoc::RubyLex.tokenize '__END__', nil
+
+    expected = [
+      @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'),
+      @TK::TkNL           .new(7, 1, 7, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_character_literal
+    tokens = RDoc::RubyLex.tokenize "?\\", nil
+
+    expected = [
+      @TK::TkCHAR.new( 0, 1,  0, "?\\"),
+      @TK::TkNL  .new( 2, 1,  2, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_def_heredoc
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+def x
+  <<E
+Line 1
+Line 2
+E
+end
+    RUBY
+
+    expected = [
+      @TK::TkDEF       .new( 0, 1,  0, 'def'),
+      @TK::TkSPACE     .new( 3, 1,  3, ' '),
+      @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
+      @TK::TkNL        .new( 5, 1,  5, "\n"),
+      @TK::TkSPACE     .new( 6, 2,  0, '  '),
+      @TK::TkHEREDOC   .new( 8, 2,  2,
+                            %Q{<<E\nLine 1\nLine 2\nE}),
+      @TK::TkNL        .new(27, 5, 28, "\n"),
+      @TK::TkEND       .new(28, 6,  0, 'end'),
+      @TK::TkNL        .new(31, 6, 28, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_hash_symbol
+    tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
+
+    expected = [
+      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkIDENTIFIER.new( 2, 1,  2, 'class'),
+      @TK::TkSYMBEG    .new( 7, 1,  7, ':'),
+      @TK::TkSTRING    .new( 8, 1,  8, '"foo"'),
+      @TK::TkSPACE     .new(13, 1, 13, ' '),
+      @TK::TkRBRACE    .new(14, 1, 14, '}'),
+      @TK::TkNL        .new(15, 1, 15, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_heredoc_CR_NL
+    tokens = RDoc::RubyLex.tokenize <<-RUBY, nil
+string = <<-STRING\r
+Line 1\r
+Line 2\r
+  STRING\r
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkASSIGN    .new( 7, 1,  7, '='),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkHEREDOC   .new( 9, 1,  9,
+                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
+      @TK::TkSPACE     .new(44, 4, 45, "\r"),
+      @TK::TkNL        .new(45, 4, 46, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_heredoc_call
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+string = <<-STRING.chomp
+Line 1
+Line 2
+  STRING
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkASSIGN    .new( 7, 1,  7, '='),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkSTRING    .new( 9, 1,  9, %Q{"Line 1\nLine 2\n"}),
+      @TK::TkDOT       .new(41, 4, 42, '.'),
+      @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'),
+      @TK::TkNL        .new(47, 4, 48, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_heredoc_indent
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+string = <<-STRING
+Line 1
+Line 2
+  STRING
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkASSIGN    .new( 7, 1,  7, '='),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkHEREDOC   .new( 9, 1,  9,
+                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
+      @TK::TkNL        .new(41, 4, 42, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_heredoc_missing_end
+    e = assert_raises RDoc::RubyLex::Error do
+      RDoc::RubyLex.tokenize <<-'RUBY', nil
+>> string1 = <<-TXT
+>" That's swell
+>" TXT
+      RUBY
+    end
+
+    assert_equal 'Missing terminating TXT for string', e.message
+  end
+
+  def test_class_tokenize_heredoc_percent_N
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+a b <<-U
+%N
+U
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
+      @TK::TkSPACE     .new( 3, 1,  3, ' '),
+      @TK::TkHEREDOC   .new( 4, 1,  4, %Q{<<-U\n%N\nU}),
+      @TK::TkNL        .new(13, 3, 14, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_identifier_high_unicode
+    tokens = RDoc::RubyLex.tokenize '𝖒', nil
+
+    expected = @TK::TkIDENTIFIER.new(0, 1, 0, '𝖒')
+
+    assert_equal expected, tokens.first
+  end
+
+  def test_class_tokenize_percent_1
+    tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new(0, 1, 0, 'v'),
+      @TK::TkMOD.new(       1, 1, 1, '%'),
+      @TK::TkINTEGER.new(   2, 1, 2, '10'),
+      @TK::TkEQ.new(        4, 1, 4, '=='),
+      @TK::TkINTEGER.new(   6, 1, 6, '10'),
+      @TK::TkNL.new(        8, 1, 8, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_r
+    tokens = RDoc::RubyLex.tokenize '%r[hi]', nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, '%r[hi]'),
+      @TK::TkNL    .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_w
+    tokens = RDoc::RubyLex.tokenize '%w[hi]', nil
+
+    expected = [
+      @TK::TkDSTRING.new( 0, 1,  0, '%w[hi]'),
+      @TK::TkNL     .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_w_quote
+    tokens = RDoc::RubyLex.tokenize '%w"hi"', nil
+
+    expected = [
+      @TK::TkDSTRING.new( 0, 1,  0, '%w"hi"'),
+      @TK::TkNL     .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_regexp
+    tokens = RDoc::RubyLex.tokenize "/hay/", nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, "/hay/"),
+      @TK::TkNL    .new( 5, 1,  5, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_regexp_options
+    tokens = RDoc::RubyLex.tokenize "/hAY/i", nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, "/hAY/i"),
+      @TK::TkNL    .new( 6, 1,  6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+
+    tokens = RDoc::RubyLex.tokenize "/hAY/ix", nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, "/hAY/ix"),
+      @TK::TkNL    .new( 7, 1,  7, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_regexp_backref
+    tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, "/[csh](..) [csh]\\1 in/"),
+      @TK::TkNL    .new(22, 1, 22, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_regexp_escape
+    tokens = RDoc::RubyLex.tokenize "/\\//", nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, "/\\//"),
+      @TK::TkNL    .new( 4, 1,  4, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_string
+    tokens = RDoc::RubyLex.tokenize "'hi'", nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1,  0, "'hi'"),
+      @TK::TkNL    .new( 4, 1,  4, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_string_escape
+    tokens = RDoc::RubyLex.tokenize '"\\n"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\n\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\r"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\r\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\f"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\f\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\\\"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\\\\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\t"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\t\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\v"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\v\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\a"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\a\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\e"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\e\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\b"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\b\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\s"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\s\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\d"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\d\""), tokens.first
+
+  end
+
+  def test_class_tokenize_string_escape_control
+    tokens = RDoc::RubyLex.tokenize '"\\C-a"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\C-a\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\c\\a"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\c\\a\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\C-\\M-a"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\C-\\M-a\""), tokens.first
+  end
+
+  def test_class_tokenize_string_escape_meta
+    tokens = RDoc::RubyLex.tokenize '"\\M-a"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\M-a\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\M-\\C-a"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\M-\\C-a\""), tokens.first
+  end
+
+  def test_class_tokenize_string_escape_hexadecimal
+    tokens = RDoc::RubyLex.tokenize '"\\x0"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x0\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\x00"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x00\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\x000"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x000\""), tokens.first
+  end
+
+  def test_class_tokenize_string_escape_octal
+    tokens = RDoc::RubyLex.tokenize '"\\0"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\0\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\00"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\00\""), tokens.first
+
+    tokens = RDoc::RubyLex.tokenize '"\\000"', nil
+    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\000\""), tokens.first
+  end
+
+  def test_class_tokenize_symbol
+    tokens = RDoc::RubyLex.tokenize 'scope module: :v1', nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'scope'),
+      @TK::TkSPACE     .new( 5, 1,  5, ' '),
+      @TK::TkIDENTIFIER.new( 6, 1,  6, 'module'),
+      @TK::TkCOLON     .new(12, 1, 12, ':'),
+      @TK::TkSPACE     .new(13, 1, 13, ' '),
+      @TK::TkSYMBEG    .new(14, 1, 14, ':'),
+      @TK::TkIDENTIFIER.new(15, 1, 15, 'v1'),
+      @TK::TkNL        .new(17, 1, 17, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_unary_minus
+    ruby_lex = RDoc::RubyLex.new("-1", nil)
+    assert_equal("-1", ruby_lex.token.value)
+
+    ruby_lex = RDoc::RubyLex.new("a[-2]", nil)
+    2.times { ruby_lex.token } # skip "a" and "["
+    assert_equal("-2", ruby_lex.token.value)
+
+    ruby_lex = RDoc::RubyLex.new("a[0..-12]", nil)
+    4.times { ruby_lex.token } # skip "a", "[", "0", and ".."
+    assert_equal("-12", ruby_lex.token.value)
+
+    ruby_lex = RDoc::RubyLex.new("0+-0.1", nil)
+    2.times { ruby_lex.token } # skip "0" and "+"
+    assert_equal("-0.1", ruby_lex.token.value)
+  end
+
+end
+
-- 
cgit v1.2.3