diff options
Diffstat (limited to 'jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb')
-rw-r--r-- | jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb | 421 |
1 files changed, 421 insertions, 0 deletions
diff --git a/jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb b/jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb new file mode 100644 index 0000000..8871b4f --- /dev/null +++ b/jni/ruby/test/rdoc/test_rdoc_ruby_lex.rb @@ -0,0 +1,421 @@ +# coding: UTF-8 + +require 'rdoc/test_case' + +class TestRDocRubyLex < RDoc::TestCase + + def setup + @TK = RDoc::RubyToken + end + + def test_class_tokenize + tokens = RDoc::RubyLex.tokenize "def x() end", nil + + expected = [ + @TK::TkDEF .new( 0, 1, 0, "def"), + @TK::TkSPACE .new( 3, 1, 3, " "), + @TK::TkIDENTIFIER.new( 4, 1, 4, "x"), + @TK::TkLPAREN .new( 5, 1, 5, "("), + @TK::TkRPAREN .new( 6, 1, 6, ")"), + @TK::TkSPACE .new( 7, 1, 7, " "), + @TK::TkEND .new( 8, 1, 8, "end"), + @TK::TkNL .new(11, 1, 11, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize___END__ + tokens = RDoc::RubyLex.tokenize '__END__', nil + + expected = [ + @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'), + @TK::TkNL .new(7, 1, 7, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_character_literal + tokens = RDoc::RubyLex.tokenize "?\\", nil + + expected = [ + @TK::TkCHAR.new( 0, 1, 0, "?\\"), + @TK::TkNL .new( 2, 1, 2, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_def_heredoc + tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil +def x + <<E +Line 1 +Line 2 +E +end + RUBY + + expected = [ + @TK::TkDEF .new( 0, 1, 0, 'def'), + @TK::TkSPACE .new( 3, 1, 3, ' '), + @TK::TkIDENTIFIER.new( 4, 1, 4, 'x'), + @TK::TkNL .new( 5, 1, 5, "\n"), + @TK::TkSPACE .new( 6, 2, 0, ' '), + @TK::TkHEREDOC .new( 8, 2, 2, + %Q{<<E\nLine 1\nLine 2\nE}), + @TK::TkNL .new(27, 5, 28, "\n"), + @TK::TkEND .new(28, 6, 0, 'end'), + @TK::TkNL .new(31, 6, 28, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_hash_symbol + tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil + + expected = [ + @TK::TkLBRACE .new( 0, 1, 0, '{'), + @TK::TkSPACE .new( 1, 1, 1, ' '), + @TK::TkIDENTIFIER.new( 2, 1, 2, 'class'), + @TK::TkSYMBEG .new( 7, 1, 7, ':'), + @TK::TkSTRING .new( 8, 1, 8, '"foo"'), + @TK::TkSPACE .new(13, 1, 13, ' '), + @TK::TkRBRACE .new(14, 1, 14, '}'), + @TK::TkNL .new(15, 1, 15, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_heredoc_CR_NL + tokens = RDoc::RubyLex.tokenize <<-RUBY, nil +string = <<-STRING\r +Line 1\r +Line 2\r + STRING\r + RUBY + + expected = [ + @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), + @TK::TkSPACE .new( 6, 1, 6, ' '), + @TK::TkASSIGN .new( 7, 1, 7, '='), + @TK::TkSPACE .new( 8, 1, 8, ' '), + @TK::TkHEREDOC .new( 9, 1, 9, + %Q{<<-STRING\nLine 1\nLine 2\n STRING}), + @TK::TkSPACE .new(44, 4, 45, "\r"), + @TK::TkNL .new(45, 4, 46, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_heredoc_call + tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil +string = <<-STRING.chomp +Line 1 +Line 2 + STRING + RUBY + + expected = [ + @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), + @TK::TkSPACE .new( 6, 1, 6, ' '), + @TK::TkASSIGN .new( 7, 1, 7, '='), + @TK::TkSPACE .new( 8, 1, 8, ' '), + @TK::TkSTRING .new( 9, 1, 9, %Q{"Line 1\nLine 2\n"}), + @TK::TkDOT .new(41, 4, 42, '.'), + @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'), + @TK::TkNL .new(47, 4, 48, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_heredoc_indent + tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil +string = <<-STRING +Line 1 +Line 2 + STRING + RUBY + + expected = [ + @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), + @TK::TkSPACE .new( 6, 1, 6, ' '), + @TK::TkASSIGN .new( 7, 1, 7, '='), + @TK::TkSPACE .new( 8, 1, 8, ' '), + @TK::TkHEREDOC .new( 9, 1, 9, + %Q{<<-STRING\nLine 1\nLine 2\n STRING}), + @TK::TkNL .new(41, 4, 42, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_heredoc_missing_end + e = assert_raises RDoc::RubyLex::Error do + RDoc::RubyLex.tokenize <<-'RUBY', nil +>> string1 = <<-TXT +>" That's swell +>" TXT + RUBY + end + + assert_equal 'Missing terminating TXT for string', e.message + end + + def test_class_tokenize_heredoc_percent_N + tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil +a b <<-U +%N +U + RUBY + + expected = [ + @TK::TkIDENTIFIER.new( 0, 1, 0, 'a'), + @TK::TkSPACE .new( 1, 1, 1, ' '), + @TK::TkIDENTIFIER.new( 2, 1, 2, 'b'), + @TK::TkSPACE .new( 3, 1, 3, ' '), + @TK::TkHEREDOC .new( 4, 1, 4, %Q{<<-U\n%N\nU}), + @TK::TkNL .new(13, 3, 14, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_identifier_high_unicode + tokens = RDoc::RubyLex.tokenize '𝖒', nil + + expected = @TK::TkIDENTIFIER.new(0, 1, 0, '𝖒') + + assert_equal expected, tokens.first + end + + def test_class_tokenize_percent_1 + tokens = RDoc::RubyLex.tokenize 'v%10==10', nil + + expected = [ + @TK::TkIDENTIFIER.new(0, 1, 0, 'v'), + @TK::TkMOD.new( 1, 1, 1, '%'), + @TK::TkINTEGER.new( 2, 1, 2, '10'), + @TK::TkEQ.new( 4, 1, 4, '=='), + @TK::TkINTEGER.new( 6, 1, 6, '10'), + @TK::TkNL.new( 8, 1, 8, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_percent_r + tokens = RDoc::RubyLex.tokenize '%r[hi]', nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, '%r[hi]'), + @TK::TkNL .new( 6, 1, 6, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_percent_w + tokens = RDoc::RubyLex.tokenize '%w[hi]', nil + + expected = [ + @TK::TkDSTRING.new( 0, 1, 0, '%w[hi]'), + @TK::TkNL .new( 6, 1, 6, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_percent_w_quote + tokens = RDoc::RubyLex.tokenize '%w"hi"', nil + + expected = [ + @TK::TkDSTRING.new( 0, 1, 0, '%w"hi"'), + @TK::TkNL .new( 6, 1, 6, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_regexp + tokens = RDoc::RubyLex.tokenize "/hay/", nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, "/hay/"), + @TK::TkNL .new( 5, 1, 5, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_regexp_options + tokens = RDoc::RubyLex.tokenize "/hAY/i", nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, "/hAY/i"), + @TK::TkNL .new( 6, 1, 6, "\n"), + ] + + assert_equal expected, tokens + + tokens = RDoc::RubyLex.tokenize "/hAY/ix", nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, "/hAY/ix"), + @TK::TkNL .new( 7, 1, 7, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_regexp_backref + tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, "/[csh](..) [csh]\\1 in/"), + @TK::TkNL .new(22, 1, 22, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_regexp_escape + tokens = RDoc::RubyLex.tokenize "/\\//", nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, "/\\//"), + @TK::TkNL .new( 4, 1, 4, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_string + tokens = RDoc::RubyLex.tokenize "'hi'", nil + + expected = [ + @TK::TkSTRING.new( 0, 1, 0, "'hi'"), + @TK::TkNL .new( 4, 1, 4, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_string_escape + tokens = RDoc::RubyLex.tokenize '"\\n"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\n\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\r"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\r\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\f"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\f\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\\\"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\\\\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\t"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\t\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\v"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\v\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\a"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\a\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\e"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\e\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\b"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\b\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\s"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\s\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\d"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\d\""), tokens.first + + end + + def test_class_tokenize_string_escape_control + tokens = RDoc::RubyLex.tokenize '"\\C-a"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\C-a\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\c\\a"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\c\\a\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\C-\\M-a"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\C-\\M-a\""), tokens.first + end + + def test_class_tokenize_string_escape_meta + tokens = RDoc::RubyLex.tokenize '"\\M-a"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\M-a\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\M-\\C-a"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\M-\\C-a\""), tokens.first + end + + def test_class_tokenize_string_escape_hexadecimal + tokens = RDoc::RubyLex.tokenize '"\\x0"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\x0\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\x00"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\x00\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\x000"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\x000\""), tokens.first + end + + def test_class_tokenize_string_escape_octal + tokens = RDoc::RubyLex.tokenize '"\\0"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\0\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\00"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\00\""), tokens.first + + tokens = RDoc::RubyLex.tokenize '"\\000"', nil + assert_equal @TK::TkSTRING.new( 0, 1, 0, "\"\\000\""), tokens.first + end + + def test_class_tokenize_symbol + tokens = RDoc::RubyLex.tokenize 'scope module: :v1', nil + + expected = [ + @TK::TkIDENTIFIER.new( 0, 1, 0, 'scope'), + @TK::TkSPACE .new( 5, 1, 5, ' '), + @TK::TkIDENTIFIER.new( 6, 1, 6, 'module'), + @TK::TkCOLON .new(12, 1, 12, ':'), + @TK::TkSPACE .new(13, 1, 13, ' '), + @TK::TkSYMBEG .new(14, 1, 14, ':'), + @TK::TkIDENTIFIER.new(15, 1, 15, 'v1'), + @TK::TkNL .new(17, 1, 17, "\n"), + ] + + assert_equal expected, tokens + end + + def test_unary_minus + ruby_lex = RDoc::RubyLex.new("-1", nil) + assert_equal("-1", ruby_lex.token.value) + + ruby_lex = RDoc::RubyLex.new("a[-2]", nil) + 2.times { ruby_lex.token } # skip "a" and "[" + assert_equal("-2", ruby_lex.token.value) + + ruby_lex = RDoc::RubyLex.new("a[0..-12]", nil) + 4.times { ruby_lex.token } # skip "a", "[", "0", and ".." + assert_equal("-12", ruby_lex.token.value) + + ruby_lex = RDoc::RubyLex.new("0+-0.1", nil) + 2.times { ruby_lex.token } # skip "0" and "+" + assert_equal("-0.1", ruby_lex.token.value) + end + +end + |