From fcbf63e62c627deae76c1b8cb8c0876c536ed811 Mon Sep 17 00:00:00 2001 From: Jari Vetoniemi Date: Mon, 16 Mar 2020 18:49:26 +0900 Subject: Fresh start --- jni/ruby/test/rdoc/test_rdoc_markup_parser.rb | 1680 +++++++++++++++++++++++++ 1 file changed, 1680 insertions(+) create mode 100644 jni/ruby/test/rdoc/test_rdoc_markup_parser.rb (limited to 'jni/ruby/test/rdoc/test_rdoc_markup_parser.rb') diff --git a/jni/ruby/test/rdoc/test_rdoc_markup_parser.rb b/jni/ruby/test/rdoc/test_rdoc_markup_parser.rb new file mode 100644 index 0000000..d27fb42 --- /dev/null +++ b/jni/ruby/test/rdoc/test_rdoc_markup_parser.rb @@ -0,0 +1,1680 @@ +# coding: utf-8 + +require 'rdoc/test_case' + +class TestRDocMarkupParser < RDoc::TestCase + + def setup + super + + @have_byteslice = ''.respond_to? :byteslice + + @RMP = @RM::Parser + end + + def test_build_heading + parser = @RMP.new + + parser.tokens.replace [ + [:TEXT, 'heading three', 4, 0], + [:NEWLINE, "\n", 17, 0], + ] + + assert_equal @RM::Heading.new(3, 'heading three'), parser.build_heading(3) + end + + def test_char_pos + parser = @RMP.new + s = parser.setup_scanner 'cät' + + s.scan(/\S+/) + + if @have_byteslice or @have_encoding then + assert_equal 3, parser.char_pos(s.pos) + else + assert_equal 4, parser.char_pos(s.pos) + end + end + + def test_get + parser = util_parser + + assert_equal [:HEADER, 1, 0, 0], parser.get + + assert_equal 7, parser.tokens.length + end + + def test_parse_bullet + str = <<-STR +* l1 +* l2 + STR + + expected = [ + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1')), + @RM::ListItem.new(nil, + @RM::Paragraph.new('l2'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_bullet_utf_8 + str = <<-STR +* 新しい機能 + STR + + expected = [ + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('新しい機能'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_bullet_verbatim_heading + str = <<-STR +* l1 + v + += H + STR + + expected = [ + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1'), + @RM::Verbatim.new("v\n"))]), + @RM::Heading.new(1, 'H')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_bullet_heading + str = <<-STR +* = l1 + STR + + expected = [ + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Heading.new(1, 'l1'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_bullet_indent + str = <<-STR +* l1 + * l1.1 +* l2 + STR + + expected = [ + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1'), + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1.1'))])), + @RM::ListItem.new(nil, + @RM::Paragraph.new('l2'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_bullet_paragraph + str = <<-STR +now is +* l1 +* l2 +the time + STR + + expected = [ + @RM::Paragraph.new('now is'), + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1')), + @RM::ListItem.new(nil, + @RM::Paragraph.new('l2')), + ]), + @RM::Paragraph.new('the time'), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_bullet_multiline + str = <<-STR +* l1 + l1+ +* l2 + STR + + expected = [ + list(:BULLET, + item(nil, + para('l1 ', 'l1+')), + item(nil, + para('l2')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_bullet_multiparagraph + str = <<-STR +* l1 + + l1+ + STR + + expected = [ + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1'), + @RM::BlankLine.new, + @RM::Paragraph.new('l1+')), + ]), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_bullet_indent_verbatim + str = <<-STR +* l1 + * l1.1 + text + code + code + + text +* l2 + STR + + expected = [ + list(:BULLET, + item(nil, + para('l1'), + list(:BULLET, + item(nil, + para('l1.1 ', 'text'), + verb("code\n", " code\n"), + para('text')))), + item(nil, + para('l2')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_dash + str = <<-STR +- one +- two + STR + + expected = [ + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('one')), + @RM::ListItem.new(nil, + @RM::Paragraph.new('two'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading + str = '= heading one' + + expected = [ + @RM::Heading.new(1, 'heading one')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_three + str = '=== heading three' + + expected = [ + @RM::Heading.new(3, 'heading three')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_bullet + str = '= * heading one' + + expected = [ + @RM::Heading.new(1, '* heading one')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_empty + str = <<-STR +=== +* bullet + STR + + expected = [ + @RM::Heading.new(3, ''), + @RM::BlankLine.new, + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('bullet'))]), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_heading + str = '= =' + + expected = [ + @RM::Heading.new(1, '=')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_lalpha + str = '= b. heading one' + + expected = [ + @RM::Heading.new(1, 'b. heading one')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_label + str = '= [heading one]' + + expected = [ + @RM::Heading.new(1, '[heading one]')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_note + str = '= heading one::' + + expected = [ + @RM::Heading.new(1, 'heading one::')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_number + str = '= 5. heading one' + + expected = [ + @RM::Heading.new(1, '5. heading one')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_heading_ualpha + str = '= B. heading one' + + expected = [ + @RM::Heading.new(1, 'B. heading one')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_label + str = <<-STR +[one] item one +[two] item two + STR + + expected = [ + list(:LABEL, + item(%w[one], + para('item one')), + item(%w[two], + para('item two')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_label_bullet + str = <<-STR +[cat] l1 + * l1.1 +[dog] l2 + STR + + expected = [ + list(:LABEL, + item(%w[cat], + para('l1'), + list(:BULLET, + item(nil, + para('l1.1')))), + item(%w[dog], + para('l2')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_label_multi_label + str = <<-STR +[one] +[two] some description + STR + + expected = [ + list(:LABEL, + item(%w[one two], + para('some description')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_label_multi_line + str = <<-STR +[cat] l1 + continuation +[dog] l2 + STR + + expected = [ + list(:LABEL, + item(%w[cat], + para('l1 ', 'continuation')), + item(%w[dog], + para('l2')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_label_newline + str = <<-STR +[one] + item one +[two] + item two + STR + + expected = [ + list(:LABEL, + item(%w[one], + para('item one')), + item(%w[two], + para('item two')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_lalpha + str = <<-STR +a. l1 +b. l2 + STR + + expected = [ + @RM::List.new(:LALPHA, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1')), + @RM::ListItem.new(nil, + @RM::Paragraph.new('l2'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_lalpha_ualpha + str = <<-STR +a. l1 +b. l2 +A. l3 +A. l4 + STR + + expected = [ + @RM::List.new(:LALPHA, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1')), + @RM::ListItem.new(nil, + @RM::Paragraph.new('l2'))]), + @RM::List.new(:UALPHA, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l3')), + @RM::ListItem.new(nil, + @RM::Paragraph.new('l4'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_lalpha_utf_8 + str = <<-STR +a. 新しい機能 + STR + + expected = [ + @RM::List.new(:LALPHA, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('新しい機能'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_line_break + str = "now is\nthe time \nfor all" + + expected = [ + para('now is ', 'the time'), + blank_line, + para('for all')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_list_list_1 + str = <<-STR +10. para 1 + + [label 1] + para 1.1 + + code + + para 1.2 + STR + + expected = [ + list(:NUMBER, + item(nil, + para('para 1'), + blank_line, + list(:LABEL, + item(%w[label\ 1], + para('para 1.1'), + blank_line, + verb("code\n"), + para('para 1.2')))))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_list_list_2 + str = <<-STR +6. para + + label 1:: text 1 + label 2:: text 2 + STR + + expected = [ + list(:NUMBER, + item(nil, + para('para'), + blank_line, + list(:NOTE, + item(%w[label\ 1], + para('text 1')), + item(%w[label\ 2], + para('text 2')))))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_list_verbatim + str = <<-STR +* one + verb1 + verb2 +* two + STR + + expected = [ + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('one'), + @RM::Verbatim.new("verb1\n", "verb2\n")), + @RM::ListItem.new(nil, + @RM::Paragraph.new('two'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_lists + str = <<-STR +now is +* l1 +1. n1 +2. n2 +* l2 +the time + STR + + expected = [ + @RM::Paragraph.new('now is'), + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1'))]), + @RM::List.new(:NUMBER, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('n1')), + @RM::ListItem.new(nil, + @RM::Paragraph.new('n2'))]), + @RM::List.new(:BULLET, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l2'))]), + @RM::Paragraph.new('the time')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_note + str = <<-STR +one:: item one +two:: item two + STR + + expected = [ + list(:NOTE, + item(%w[one], + para('item one')), + item(%w[two], + para('item two')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_note_empty + str = <<-STR +one:: +two:: + STR + + expected = [ + list(:NOTE, + item(%w[one two], + blank_line))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_note_note + str = <<-STR +one:: two:: + STR + + expected = [ + list(:NOTE, + item(%w[one], + list(:NOTE, + item(%w[two], + blank_line))))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_number_bullet + str = <<-STR +1. l1 + * l1.1 +2. l2 + STR + + expected = [ + list(:NUMBER, + item(nil, + para('l1'), + list(:BULLET, + item(nil, + para('l1.1')))), + item(nil, + para('l2')))] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_paragraph + str = <<-STR +now is the time + +for all good men + STR + + expected = [ + @RM::Paragraph.new('now is the time'), + @RM::BlankLine.new, + @RM::Paragraph.new('for all good men')] + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_paragraph_multiline + str = "now is the time\nfor all good men" + + expected = @RM::Paragraph.new 'now is the time ', 'for all good men' + assert_equal [expected], @RMP.parse(str).parts + end + + def test_parse_paragraph_verbatim + str = <<-STR +now is the time + code _line_ here +for all good men + STR + + expected = [ + @RM::Paragraph.new('now is the time'), + @RM::Verbatim.new("code _line_ here\n"), + @RM::Paragraph.new('for all good men'), + ] + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_rule + str = <<-STR +now is the time + +--- + +for all good men + STR + + expected = [ + @RM::Paragraph.new('now is the time'), + @RM::BlankLine.new, + @RM::Rule.new(1), + @RM::BlankLine.new, + @RM::Paragraph.new('for all good men')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_ualpha + str = <<-STR +A. l1 +B. l2 + STR + + expected = [ + @RM::List.new(:UALPHA, *[ + @RM::ListItem.new(nil, + @RM::Paragraph.new('l1')), + @RM::ListItem.new(nil, + @RM::Paragraph.new('l2'))])] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_trailing_cr + expected = [ @RM::Paragraph.new('Text') ] + # FIXME hangs the parser: + assert_equal expected, @RMP.parse("Text\r").parts + end + + def test_parse_verbatim + str = <<-STR +now is + code +the time + STR + + expected = [ + @RM::Paragraph.new('now is'), + @RM::Verbatim.new("code\n"), + @RM::Paragraph.new('the time'), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_bullet + str = <<-STR + * blah + STR + + expected = [ + @RM::Verbatim.new("* blah\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_dash + str = <<-STR + - blah + STR + + expected = [ + @RM::Verbatim.new("- blah\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_fold + str = <<-STR +now is + code + + + code1 + +the time + STR + + expected = [ + @RM::Paragraph.new('now is'), + @RM::Verbatim.new("code\n", "\n", "code1\n"), + @RM::Paragraph.new('the time'), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_heading + str = <<-STR +text + === heading three + STR + + expected = [ + @RM::Paragraph.new('text'), + @RM::Verbatim.new("=== heading three\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_heading2 + str = "text\n code\n=== heading three" + + expected = [ + @RM::Paragraph.new('text'), + @RM::Verbatim.new("code\n"), + @RM::Heading.new(3, 'heading three')] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_label + str = <<-STR + [blah] blah + STR + + expected = [ + @RM::Verbatim.new("[blah] blah\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_lalpha + str = <<-STR + b. blah + STR + + expected = [ + @RM::Verbatim.new("b. blah\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_markup_example + str = <<-STR +text + code + === heading three + STR + + expected = [ + @RM::Paragraph.new('text'), + @RM::Verbatim.new("code\n", "=== heading three\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_merge + str = <<-STR +now is + code + + code1 +the time + STR + + expected = [ + @RM::Paragraph.new('now is'), + @RM::Verbatim.new("code\n", "\n", "code1\n"), + @RM::Paragraph.new('the time'), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_merge2 + str = <<-STR +now is + code + + code1 + + code2 +the time + STR + + expected = [ + @RM::Paragraph.new('now is'), + @RM::Verbatim.new("code\n", "\n", "code1\n", "\n", "code2\n"), + @RM::Paragraph.new('the time'), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_multiline + str = <<-STR +now is + code + code1 +the time + STR + + expected = [ + @RM::Paragraph.new('now is'), + @RM::Verbatim.new("code\n", "code1\n"), + @RM::Paragraph.new('the time'), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_multilevel + str = <<-STR +now is the time + code + more code +for all good men + STR + + expected = [ + @RM::Paragraph.new('now is the time'), + @RM::Verbatim.new(" code\n", + "more code\n"), + @RM::Paragraph.new('for all good men'), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_note + str = <<-STR + blah:: blah + STR + + expected = [ + @RM::Verbatim.new("blah:: blah\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_number + str = <<-STR + 2. blah + STR + + expected = [ + @RM::Verbatim.new("2. blah\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_rule + str = <<-STR +text + + --- lib/blah.rb.orig + +++ lib/blah.rb + STR + + expected = [ + @RM::Paragraph.new('text'), + @RM::BlankLine.new, + @RM::Verbatim.new("--- lib/blah.rb.orig\n", + "+++ lib/blah.rb\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_rule2 + str = <<-STR.chomp +text + + --- + STR + + expected = [ + @RM::Paragraph.new('text'), + @RM::BlankLine.new, + @RM::Verbatim.new("---")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_trim + str = <<-STR +now is + code + + code1 + +the time + STR + + expected = [ + @RM::Paragraph.new('now is'), + @RM::Verbatim.new("code\n", + "\n", + "code1\n"), + @RM::Paragraph.new('the time'), + ] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_verbatim_ualpha + str = <<-STR + B. blah + STR + + expected = [ + @RM::Verbatim.new("B. blah\n")] + + assert_equal expected, @RMP.parse(str).parts + end + + def test_parse_whitespace + expected = [ + @RM::Paragraph.new('hello'), + ] + + assert_equal expected, @RMP.parse('hello').parts + + expected = [ + @RM::Verbatim.new('hello '), + ] + + assert_equal expected, @RMP.parse(' hello ').parts + + expected = [ + @RM::Verbatim.new('hello '), + ] + + assert_equal expected, @RMP.parse(' hello ').parts + + expected = [ + @RM::Paragraph.new('1'), + @RM::Verbatim.new("2\n", ' 3'), + ] + + assert_equal expected, @RMP.parse("1\n 2\n 3").parts + + expected = [ + @RM::Verbatim.new("1\n", + " 2\n", + " 3"), + ] + + assert_equal expected, @RMP.parse(" 1\n 2\n 3").parts + + expected = [ + @RM::Paragraph.new('1'), + @RM::Verbatim.new("2\n", + " 3\n"), + @RM::Paragraph.new('1'), + @RM::Verbatim.new('2'), + ] + + assert_equal expected, @RMP.parse("1\n 2\n 3\n1\n 2").parts + + expected = [ + @RM::Verbatim.new("1\n", + " 2\n", + " 3\n", + "1\n", + ' 2'), + ] + + assert_equal expected, @RMP.parse(" 1\n 2\n 3\n 1\n 2").parts + + expected = [ + @RM::Verbatim.new("1\n", + " 2\n", + "\n", + ' 3'), + ] + + assert_equal expected, @RMP.parse(" 1\n 2\n\n 3").parts + end + + def test_peek_token + parser = util_parser + + assert_equal [:HEADER, 1, 0, 0], parser.peek_token + + assert_equal 8, parser.tokens.length + end + + def test_skip + parser = util_parser + + assert_equal [:HEADER, 1, 0, 0], parser.skip(:HEADER) + + assert_equal [:TEXT, 'Heading', 2, 0], parser.get + + assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token + + assert_raises RDoc::Markup::Parser::ParseError do + parser.skip :NONE + end + + assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token + + assert_equal nil, parser.skip(:NONE, false) + + assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token + end + + def test_tokenize_bullet + str = <<-STR +* l1 + STR + + expected = [ + [:BULLET, '*', 0, 0], + [:TEXT, 'l1', 2, 0], + [:NEWLINE, "\n", 4, 0], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_bullet_indent + str = <<-STR +* l1 + * l1.1 + STR + + expected = [ + [:BULLET, '*', 0, 0], + [:TEXT, 'l1', 2, 0], + [:NEWLINE, "\n", 4, 0], + [:BULLET, '*', 2, 1], + [:TEXT, 'l1.1', 4, 1], + [:NEWLINE, "\n", 8, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_heading + str = <<-STR += Heading +== Heading 2 + STR + + expected = [ + [:HEADER, 1, 0, 0], + [:TEXT, 'Heading', 2, 0], + [:NEWLINE, "\n", 9, 0], + [:HEADER, 2, 0, 1], + [:TEXT, 'Heading 2', 3, 1], + [:NEWLINE, "\n", 12, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_heading_empty + str = <<-STR +=== +* bullet + STR + + expected = [ + [:HEADER, 3, 0, 0], + [:NEWLINE, "\n", 3, 0], + [:BULLET, "*", 0, 1], + [:TEXT, "bullet", 2, 1], + [:NEWLINE, "\n", 8, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_heading_heading + str = <<-STR += = + STR + + expected = [ + [:HEADER, 1, 0, 0], + [:TEXT, '=', 2, 0], + [:NEWLINE, "\n", 3, 0], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_heading_no_space + str = <<-STR +=Heading +==Heading 2 + STR + + expected = [ + [:HEADER, 1, 0, 0], + [:TEXT, 'Heading', 1, 0], + [:NEWLINE, "\n", 8, 0], + [:HEADER, 2, 0, 1], + [:TEXT, 'Heading 2', 2, 1], + [:NEWLINE, "\n", 11, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_label + str = <<-STR +[cat] l1 +[dog] l1.1 + STR + + expected = [ + [:LABEL, 'cat', 0, 0], + [:TEXT, 'l1', 6, 0], + [:NEWLINE, "\n", 8, 0], + [:LABEL, 'dog', 0, 1], + [:TEXT, 'l1.1', 6, 1], + [:NEWLINE, "\n", 10, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_label_note + str = <<-STR +[label] + note:: + STR + + expected = [ + [:LABEL, 'label', 0, 0], + [:NEWLINE, "\n", 7, 0], + [:NOTE, 'note', 2, 1], + [:NEWLINE, "\n", 8, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_label_newline + str = <<-STR +[cat] + l1 + STR + + expected = [ + [:LABEL, 'cat', 0, 0], + [:NEWLINE, "\n", 5, 0], + [:TEXT, 'l1', 2, 1], + [:NEWLINE, "\n", 4, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_label_newline_windows + str = <<-STR +[cat]\r + l1\r + STR + + expected = [ + [:LABEL, 'cat', 0, 0], + [:NEWLINE, "\n", 6, 0], + [:TEXT, 'l1', 2, 1], + [:NEWLINE, "\n", 5, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_lalpha + str = <<-STR +a. l1 +b. l1.1 + STR + + expected = [ + [:LALPHA, 'a', 0, 0], + [:TEXT, 'l1', 3, 0], + [:NEWLINE, "\n", 5, 0], + [:LALPHA, 'b', 0, 1], + [:TEXT, 'l1.1', 3, 1], + [:NEWLINE, "\n", 7, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_line_break + str = "now is\nthe time \nfor all\n" + + expected = [ + [:TEXT, 'now is', 0, 0], + [:NEWLINE, "\n", 6, 0], + [:TEXT, 'the time', 0, 1], + [:BREAK, " ", 8, 1], + [:NEWLINE, "\n", 10, 1], + [:TEXT, 'for all', 0, 2], + [:NEWLINE, "\n", 7, 2], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_line_break_long + str = "now is\nthe time \nfor all\n" + + expected = [ + [:TEXT, 'now is', 0, 0], + [:NEWLINE, "\n", 6, 0], + [:TEXT, 'the time ', 0, 1], + [:BREAK, ' ', 9, 1], + [:NEWLINE, "\n", 11, 1], + [:TEXT, 'for all', 0, 2], + [:NEWLINE, "\n", 7, 2], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_line_break_no_short + str = "now is\nthe time \nfor all\n" + + expected = [ + [:TEXT, 'now is', 0, 0], + [:NEWLINE, "\n", 6, 0], + [:TEXT, 'the time ', 0, 1], + [:NEWLINE, "\n", 9, 1], + [:TEXT, 'for all', 0, 2], + [:NEWLINE, "\n", 7, 2], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_note + str = <<-STR +cat:: l1 +dog:: l1.1 + STR + + expected = [ + [:NOTE, 'cat', 0, 0], + [:TEXT, 'l1', 6, 0], + [:NEWLINE, "\n", 8, 0], + [:NOTE, 'dog', 0, 1], + [:TEXT, 'l1.1', 6, 1], + [:NEWLINE, "\n", 10, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_note_empty + str = <<-STR +cat:: +dog:: + STR + + expected = [ + [:NOTE, 'cat', 0, 0], + [:NEWLINE, "\n", 5, 0], + [:NOTE, 'dog', 0, 1], + [:NEWLINE, "\n", 5, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_note_newline + str = <<-STR +cat:: + l1 + STR + + expected = [ + [:NOTE, 'cat', 0, 0], + [:NEWLINE, "\n", 5, 0], + [:TEXT, 'l1', 2, 1], + [:NEWLINE, "\n", 4, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_note_utf_8 + skip 'Encoding not implemented' unless @have_encoding + + str = <<-STR +cät:: l1a + l1b +døg:: l2a + l2b + STR + + expected = [ + [:NOTE, 'cät', 0, 0], + [:TEXT, 'l1a', 6, 0], + [:NEWLINE, "\n", 9, 0], + [:TEXT, 'l1b', 6, 1], + [:NEWLINE, "\n", 9, 1], + [:NOTE, 'døg', 0, 2], + [:TEXT, 'l2a', 6, 2], + [:NEWLINE, "\n", 9, 2], + [:TEXT, 'l2b', 6, 3], + [:NEWLINE, "\n", 9, 3], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_note_newline_windows + str = <<-STR +cat::\r + l1\r + STR + + expected = [ + [:NOTE, 'cat', 0, 0], + [:NEWLINE, "\n", 6, 0], + [:TEXT, 'l1', 2, 1], + [:NEWLINE, "\n", 5, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_note_not + str = <<-STR +Cat::Dog + STR + + expected = [ + [:TEXT, 'Cat::Dog', 0, 0], + [:NEWLINE, "\n", 8, 0], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_number + str = <<-STR +1. l1 +2. l1.1 + STR + + expected = [ + [:NUMBER, '1', 0, 0], + [:TEXT, 'l1', 3, 0], + [:NEWLINE, "\n", 5, 0], + [:NUMBER, '2', 0, 1], + [:TEXT, 'l1.1', 3, 1], + [:NEWLINE, "\n", 7, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_number_period + str = <<-STR +1. blah blah blah + l. +2. blah blah blah blah + d. + STR + + expected = [ + [:NUMBER, "1", 0, 0], + [:TEXT, "blah blah blah", 3, 0], + [:NEWLINE, "\n", 17, 0], + + [:TEXT, "l.", 3, 1], + [:NEWLINE, "\n", 5, 1], + + [:NUMBER, "2", 0, 2], + [:TEXT, "blah blah blah blah", 3, 2], + [:NEWLINE, "\n", 22, 2], + + [:TEXT, "d.", 3, 3], + [:NEWLINE, "\n", 5, 3] + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_number_period_continue + str = <<-STR +1. blah blah blah + l. more stuff +2. blah blah blah blah + d. other stuff + STR + + expected = [ + [:NUMBER, "1", 0, 0], + [:TEXT, "blah blah blah", 3, 0], + [:NEWLINE, "\n", 17, 0], + + [:LALPHA, "l", 3, 1], + [:TEXT, "more stuff", 7, 1], + [:NEWLINE, "\n", 17, 1], + + [:NUMBER, "2", 0, 2], + [:TEXT, "blah blah blah blah", 3, 2], + [:NEWLINE, "\n", 22, 2], + + [:LALPHA, "d", 3, 3], + [:TEXT, "other stuff", 6, 3], + [:NEWLINE, "\n", 17, 3] + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_paragraphs + str = <<-STR +now is +the time + +for all + STR + + expected = [ + [:TEXT, 'now is', 0, 0], + [:NEWLINE, "\n", 6, 0], + [:TEXT, 'the time', 0, 1], + [:NEWLINE, "\n", 8, 1], + [:NEWLINE, "\n", 0, 2], + [:TEXT, 'for all', 0, 3], + [:NEWLINE, "\n", 7, 3], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_rule + str = <<-STR +--- + +--- blah --- + STR + + expected = [ + [:RULE, 1, 0, 0], + [:NEWLINE, "\n", 3, 0], + [:NEWLINE, "\n", 0, 1], + [:TEXT, "--- blah ---", 0, 2], + [:NEWLINE, "\n", 12, 2], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_rule_windows + str = <<-STR +---\r + +--- blah ---\r + STR + + expected = [ + [:RULE, 1, 0, 0], + [:NEWLINE, "\n", 4, 0], + [:NEWLINE, "\n", 0, 1], + [:TEXT, "--- blah ---", 0, 2], + [:NEWLINE, "\n", 13, 2], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_ualpha + str = <<-STR +A. l1 +B. l1.1 + STR + + expected = [ + [:UALPHA, 'A', 0, 0], + [:TEXT, 'l1', 3, 0], + [:NEWLINE, "\n", 5, 0], + [:UALPHA, 'B', 0, 1], + [:TEXT, 'l1.1', 3, 1], + [:NEWLINE, "\n", 7, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_verbatim_heading + str = <<-STR +Example heading: + + === heading three + STR + + expected = [ + [:TEXT, 'Example heading:', 0, 0], + [:NEWLINE, "\n", 16, 0], + [:NEWLINE, "\n", 0, 1], + [:HEADER, 3, 3, 2], + [:TEXT, 'heading three', 7, 2], + [:NEWLINE, "\n", 20, 2], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_verbatim_rule + str = <<-STR + Verbatim section here that is double-underlined + =============================================== + STR + + expected = [ + [:TEXT, 'Verbatim section here that is double-underlined', 2, 0], + [:NEWLINE, "\n", 49, 0], + [:HEADER, 47, 2, 1], + [:NEWLINE, "\n", 49, 1], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_tokenize_verbatim_rule_fancy + str = <<-STR + A + b + =============================================== + c + STR + + expected = [ + [:TEXT, 'A', 2, 0], + [:NEWLINE, "\n", 3, 0], + [:TEXT, 'b', 4, 1], + [:NEWLINE, "\n", 5, 1], + [:HEADER, 47, 2, 2], + [:NEWLINE, "\n", 49, 2], + [:TEXT, 'c', 4, 3], + [:NEWLINE, "\n", 5, 3], + ] + + assert_equal expected, @RMP.tokenize(str) + end + + def test_token_pos + parser = @RMP.new + s = parser.setup_scanner 'cät' + + s.scan(/\S+/) + + if @have_encoding or @have_byteslice then + assert_equal [3, 0], parser.token_pos(s.pos) + else + assert_equal [4, 0], parser.token_pos(s.pos) + end + end + + # HACK move to Verbatim test case + def test_verbatim_normalize + v = @RM::Verbatim.new "foo\n", "\n", "\n", "bar\n" + + v.normalize + + assert_equal ["foo\n", "\n", "bar\n"], v.parts + + v = @RM::Verbatim.new "foo\n", "\n" + + v.normalize + + assert_equal ["foo\n"], v.parts + end + + def test_unget + parser = util_parser + + parser.get + + parser.unget + + assert_equal [:HEADER, 1, 0, 0], parser.peek_token + + assert_raises @RMP::Error do + parser.unget + end + + assert_equal 8, parser.tokens.length + end + + def util_parser + str = <<-STR += Heading + +Some text here +some more text over here + STR + + @parser = @RMP.new + @parser.tokenize str + @parser + end + +end + -- cgit v1.2.3