Fresh start

7 files changed, 2574 insertions, 0 deletions

diff --git a/jni/ruby/test/ripper/dummyparser.rb b/jni/ruby/test/ripper/dummyparser.rb
new file mode 100644
index 0000000..35c08b5
--- /dev/null
+++ b/jni/ruby/test/ripper/dummyparser.rb
@@ -0,0 +1,220 @@
+#
+# dummyparser.rb
+#
+
+require 'ripper'
+
+class Node
+  def initialize(name, *nodes)
+    @name = name
+    @children = nodes
+  end
+
+  attr_reader :name, :children
+
+  def to_s
+    "#{@name}(#{Node.trim_nil(@children).map {|n| n.to_s }.join(',')})"
+  end
+
+  def self.trim_nil(list)
+    if !list.empty? and list.last.nil?
+      list = list[0...-1]
+      list.pop while !list.empty? and list.last.nil?
+    end
+    list
+  end
+end
+
+class NodeList
+  def initialize(list = [])
+    @list = list
+  end
+
+  attr_reader :list
+
+  def push(item)
+    @list.push item
+    self
+  end
+
+  def prepend(items)
+    @list.unshift items
+  end
+
+  def to_s
+    "[#{@list.join(',')}]"
+  end
+end
+
+class DummyParser < Ripper
+  def hook(*names)
+    class << self; self; end.class_eval do
+      names.each do |name|
+        define_method(name) do |*a, &b|
+          result = super(*a, &b)
+          yield(name, *a)
+          result
+        end
+      end
+    end
+    self
+  end
+
+  def on_program(stmts)
+    stmts
+  end
+
+  def on_stmts_new
+    NodeList.new
+  end
+
+  def on_stmts_add(stmts, st)
+    stmts.push st
+    stmts
+  end
+
+  def on_void_stmt
+    Node.new('void')
+  end
+
+  def on_var_ref(name)
+    Node.new('ref', name)
+  end
+
+  def on_var_alias(a, b)
+    Node.new('valias', a, b)
+  end
+
+  def on_alias_error(a)
+    Node.new('aliaserr', a)
+  end
+
+  def on_arg_paren(args)
+    args
+  end
+
+  def on_args_new
+    NodeList.new
+  end
+
+  def on_args_add(list, arg)
+    list.push(arg)
+  end
+
+  def on_args_add_block(list, blk)
+    if blk
+      list.push('&' + blk.to_s)
+    else
+      list
+    end
+  end
+
+  def on_args_add_star(list, arg)
+    list.push('*' + arg.to_s)
+  end
+
+  def on_args_prepend(list, args)
+    list.prepend args
+    list
+  end
+
+  def on_method_add_arg(m, arg)
+    if arg == nil
+      arg = on_args_new
+    end
+    m.children.push arg
+    m
+  end
+
+  def on_method_add_block(m, b)
+    on_args_add_block(m.children, b)
+    m
+  end
+
+  def on_paren(params)
+    params
+  end
+
+  def on_brace_block(params, code)
+    Node.new('block', params, code)
+  end
+
+  def on_block_var(params, shadow)
+    params
+  end
+
+  def on_rest_param(var)
+    "*#{var}"
+  end
+
+  def on_blockarg(var)
+    "&#{var}"
+  end
+
+  def on_params(required, optional, rest, more, keyword, keyword_rest, block)
+    args = NodeList.new
+
+    required.each do |req|
+      args.push(req)
+    end if required
+
+    optional.each do |var, val|
+      args.push("#{var}=#{val}")
+    end if optional
+
+    args.push(rest) if rest
+
+    more.each do |m|
+      args.push(m)
+    end if more
+
+    args.push(block) if block
+    args
+  end
+
+  def on_assoc_new(a, b)
+    Node.new('assoc', a, b)
+  end
+
+  def on_bare_assoc_hash(assoc_list)
+    Node.new('assocs', *assoc_list)
+  end
+
+  def on_assoclist_from_args(a)
+    Node.new('assocs', *a)
+  end
+
+  def on_word_new
+    ""
+  end
+
+  def on_word_add(word, w)
+    word << w
+  end
+
+  def on_words_new
+    NodeList.new
+  end
+
+  def on_words_add(words, word)
+    words.push word
+  end
+
+  def on_qwords_new
+    NodeList.new
+  end
+
+  def on_qwords_add(words, word)
+    words.push word
+  end
+
+  def on_rescue(exc, *rest)
+    Node.new('rescue', (exc && NodeList.new(exc)), *rest)
+  end
+
+  (Ripper::PARSER_EVENTS.map(&:to_s) - instance_methods(false).map {|n|n.to_s.sub(/^on_/, '')}).each do |event|
+    define_method(:"on_#{event}") do |*args|
+      Node.new(event, *args)
+    end
+  end
+end
diff --git a/jni/ruby/test/ripper/test_files.rb b/jni/ruby/test/ripper/test_files.rb
new file mode 100644
index 0000000..5541a09
--- /dev/null
+++ b/jni/ruby/test/ripper/test_files.rb
@@ -0,0 +1,23 @@
+require 'test/unit'
+
+module TestRipper; end
+class TestRipper::Generic < Test::Unit::TestCase
+  def test_parse_files
+    srcdir = File.expand_path("../../..", __FILE__)
+    assert_separately(%W[--disable-gem -rripper - #{srcdir}],
+                      __FILE__, __LINE__, <<-'eom', timeout: Float::INFINITY)
+      TEST_RATIO = 0.05 # testing all files needs too long time...
+      class Parser < Ripper
+        PARSER_EVENTS.each {|n| eval "def on_#{n}(*args) r = [:#{n}, *args]; r.inspect; Object.new end" }
+        SCANNER_EVENTS.each {|n| eval "def on_#{n}(*args) r = [:#{n}, *args]; r.inspect; Object.new end" }
+      end
+      dir = ARGV.shift
+      for script in Dir["#{dir}/{lib,sample,ext,test}/**/*.rb"].sort
+        next if TEST_RATIO < rand
+        assert_nothing_raised("ripper failed to parse: #{script.inspect}") {
+          Parser.new(File.read(script), script).parse
+        }
+      end
+    eom
+  end
+end
diff --git a/jni/ruby/test/ripper/test_filter.rb b/jni/ruby/test/ripper/test_filter.rb
new file mode 100644
index 0000000..005a5d8
--- /dev/null
+++ b/jni/ruby/test/ripper/test_filter.rb
@@ -0,0 +1,83 @@
+begin
+  require 'ripper'
+  require 'test/unit'
+  ripper_test = true
+  module TestRipper; end
+rescue LoadError
+end
+
+class TestRipper::Filter < Test::Unit::TestCase
+
+  class Filter < Ripper::Filter
+    def on_default(event, token, data)
+      if data.empty?
+        data[:filename] = filename rescue nil
+        data[:lineno] = lineno
+        data[:column] = column
+        data[:token] = token
+      end
+      data
+    end
+  end
+
+  def filename
+    File.expand_path(__FILE__)
+  end
+
+  def test_filter_filename_unset
+    data = {}
+    filter = Filter.new(File.read(filename))
+    filter.parse(data)
+    assert_equal('-', data[:filename], "[ruby-dev:37856]")
+    assert_equal('-', filter.filename)
+  end
+
+  def test_filter_filename
+    data = {}
+    filter = Filter.new(File.read(filename), filename)
+    assert_equal(filename, filter.filename)
+    filter.parse(data)
+    assert_equal(filename, data[:filename])
+    assert_equal(filename, filter.filename)
+  end
+
+  def test_filter_lineno
+    data = {}
+    src = File.read(filename)
+    src_lines = src.count("\n")
+    filter = Filter.new(src)
+    assert_equal(nil, filter.lineno)
+    filter.parse(data)
+    assert_equal(1, data[:lineno])
+    assert_equal(src_lines, filter.lineno)
+  end
+
+  def test_filter_lineno_set
+    data = {}
+    src = File.read(filename)
+    src_lines = src.count("\n")
+    filter = Filter.new(src, '-', 100)
+    assert_equal(nil, filter.lineno)
+    filter.parse(data)
+    assert_equal(100, data[:lineno])
+    assert_equal(src_lines+100-1, filter.lineno)
+  end
+
+  def test_filter_column
+    data = {}
+    src = File.read(filename)
+    last_columns = src[/(.*)\Z/].size
+    filter = Filter.new(src)
+    assert_equal(nil, filter.column)
+    filter.parse(data)
+    assert_equal(0, data[:column])
+    assert_equal(last_columns, filter.column)
+  end
+
+  def test_filter_token
+    data = {}
+    filter = Filter.new(File.read(filename))
+    filter.parse(data)
+    assert_equal("begin", data[:token])
+  end
+end if ripper_test
diff --git a/jni/ruby/test/ripper/test_parser_events.rb b/jni/ruby/test/ripper/test_parser_events.rb
new file mode 100644
index 0000000..08296a2
--- /dev/null
+++ b/jni/ruby/test/ripper/test_parser_events.rb
@@ -0,0 +1,1232 @@
+begin
+  require_relative 'dummyparser'
+  require 'test/unit'
+  ripper_test = true
+  module TestRipper; end
+rescue LoadError
+end
+
+class TestRipper::ParserEvents < Test::Unit::TestCase
+
+  def test_event_coverage
+    dispatched = Ripper::PARSER_EVENTS
+    tested = self.class.instance_methods(false).grep(/\Atest_(\w+)/) {$1.intern}
+    assert_empty dispatched-tested
+  end
+
+  def parse(str, nm = nil, &bl)
+    dp = DummyParser.new(str)
+    dp.hook(*nm, &bl) if nm
+    dp.parse.to_s
+  end
+
+  def compile_error(str)
+    parse(str, :compile_error) {|e, msg| return msg}
+  end
+
+  def test_program
+    thru_program = false
+    assert_equal '[void()]', parse('', :on_program) {thru_program = true}
+    assert_equal true, thru_program
+  end
+
+  def test_stmts_new
+    assert_equal '[void()]', parse('')
+  end
+
+  def test_stmts_add
+    assert_equal '[ref(nil)]', parse('nil')
+    assert_equal '[ref(nil),ref(nil)]', parse('nil;nil')
+    assert_equal '[ref(nil),ref(nil),ref(nil)]', parse('nil;nil;nil')
+  end
+
+  def test_void_stmt
+    assert_equal '[void()]', parse('')
+    assert_equal '[void()]', parse('; ;')
+  end
+
+  def test_var_ref
+    assert_equal '[assign(var_field(a),ref(a))]', parse('a=a')
+    assert_equal '[ref(nil)]', parse('nil')
+    assert_equal '[ref(true)]', parse('true')
+  end
+
+  def test_vcall
+    assert_equal '[vcall(a)]', parse('a')
+  end
+
+  def test_BEGIN
+    assert_equal '[BEGIN([void()])]', parse('BEGIN{}')
+    assert_equal '[BEGIN([ref(nil)])]', parse('BEGIN{nil}')
+  end
+
+  def test_END
+    assert_equal '[END([void()])]', parse('END{}')
+    assert_equal '[END([ref(nil)])]', parse('END{nil}')
+  end
+
+  def test_alias
+    assert_equal '[alias(symbol_literal(a),symbol_literal(b))]', parse('alias a b')
+  end
+
+  def test_var_alias
+    assert_equal '[valias($a,$g)]', parse('alias $a $g')
+  end
+
+  def test_alias_error
+    assert_equal '[aliaserr(valias($a,$1))]', parse('alias $a $1')
+  end
+
+  def test_arglist
+    assert_equal '[fcall(m,[])]', parse('m()')
+    assert_equal '[fcall(m,[1])]', parse('m(1)')
+    assert_equal '[fcall(m,[1,2])]', parse('m(1,2)')
+    assert_equal '[fcall(m,[*vcall(r)])]', parse('m(*r)')
+    assert_equal '[fcall(m,[1,*vcall(r)])]', parse('m(1,*r)')
+    assert_equal '[fcall(m,[1,2,*vcall(r)])]', parse('m(1,2,*r)')
+    assert_equal '[fcall(m,[&vcall(r)])]', parse('m(&r)')
+    assert_equal '[fcall(m,[1,&vcall(r)])]', parse('m(1,&r)')
+    assert_equal '[fcall(m,[1,2,&vcall(r)])]', parse('m(1,2,&r)')
+    assert_equal '[fcall(m,[*vcall(a),&vcall(b)])]', parse('m(*a,&b)')
+    assert_equal '[fcall(m,[1,*vcall(a),&vcall(b)])]', parse('m(1,*a,&b)')
+    assert_equal '[fcall(m,[1,2,*vcall(a),&vcall(b)])]', parse('m(1,2,*a,&b)')
+  end
+
+  def test_args_add
+    thru_args_add = false
+    parse('m(a)', :on_args_add) {thru_args_add = true}
+    assert_equal true, thru_args_add
+  end
+
+  def test_args_add_block
+    thru_args_add_block = false
+    parse('m(&b)', :on_args_add_block) {thru_args_add_block = true}
+    assert_equal true, thru_args_add_block
+  end
+
+  def test_args_add_star
+    thru_args_add_star = false
+    parse('m(*a)', :on_args_add_star) {thru_args_add_star = true}
+    assert_equal true, thru_args_add_star
+    thru_args_add_star = false
+    parse('m(*a, &b)', :on_args_add_star) {thru_args_add_star = true}
+    assert_equal true, thru_args_add_star
+  end
+
+  def test_args_new
+    thru_args_new = false
+    parse('m()', :on_args_new) {thru_args_new = true}
+    assert_equal true, thru_args_new
+  end
+
+  def test_arg_paren
+    # FIXME
+  end
+
+  def test_aref
+    assert_equal '[aref(vcall(v),[1])]', parse('v[1]')
+    assert_equal '[aref(vcall(v),[1,2])]', parse('v[1,2]')
+  end
+
+  def test_assoclist_from_args
+    thru_assoclist_from_args = false
+    parse('{a=>b}', :on_assoclist_from_args) {thru_assoclist_from_args = true}
+    assert_equal true, thru_assoclist_from_args
+  end
+
+  def test_assocs
+    assert_equal '[fcall(m,[assocs(assoc(1,2))])]', parse('m(1=>2)')
+    assert_equal '[fcall(m,[assocs(assoc(1,2),assoc(3,4))])]', parse('m(1=>2,3=>4)')
+    assert_equal '[fcall(m,[3,assocs(assoc(1,2))])]', parse('m(3,1=>2)')
+  end
+
+  def test_assoc_new
+    thru_assoc_new = false
+    parse('{a=>b}', :on_assoc_new) {thru_assoc_new = true}
+    assert_equal true, thru_assoc_new
+  end
+
+  def test_assoc_splat
+    thru_assoc_splat = false
+    parse('m(**h)', :on_assoc_splat) {thru_assoc_splat = true}
+    assert_equal true, thru_assoc_splat
+  end
+
+  def test_aref_field
+    assert_equal '[assign(aref_field(vcall(a),[1]),2)]', parse('a[1]=2')
+  end
+
+  def test_arg_ambiguous
+    thru_arg_ambiguous = false
+    parse('m //', :on_arg_ambiguous) {thru_arg_ambiguous = true}
+    assert_equal true, thru_arg_ambiguous
+  end
+
+  def test_operator_ambiguous
+    thru_operator_ambiguous = false
+    parse('a=1; a %[]', :on_operator_ambiguous) {thru_operator_ambiguous = true}
+    assert_equal true, thru_operator_ambiguous
+  end
+
+  def test_array   # array literal
+    assert_equal '[array([1,2,3])]', parse('[1,2,3]')
+    assert_equal '[array([abc,def])]', parse('%w[abc def]')
+    assert_equal '[array([abc,def])]', parse('%W[abc def]')
+  end
+
+  def test_assign   # generic assignment
+    assert_equal '[assign(var_field(v),1)]', parse('v=1')
+  end
+
+  def test_assign_error
+    # for test_coverage
+  end
+
+  def test_assign_error_backref
+    thru_assign_error = false
+    parse('$` = 1', :on_assign_error) {thru_assign_error = true}
+    assert_equal true, thru_assign_error
+    thru_assign_error = false
+    parse('$`, _ = 1', :on_assign_error) {thru_assign_error = true}
+    assert_equal true, thru_assign_error
+  end
+
+  def test_assign_error_const_qualified
+    thru_assign_error = false
+    parse('self::X = 1', :on_assign_error) {thru_assign_error = true}
+    assert_equal false, thru_assign_error
+    parse("def m\n self::X = 1\nend", :on_assign_error) {thru_assign_error = true}
+    assert_equal true, thru_assign_error
+    thru_assign_error = false
+    parse("def m\n self::X, a = 1, 2\nend", :on_assign_error) {thru_assign_error = true}
+    assert_equal true, thru_assign_error
+  end
+
+  def test_assign_error_const
+    thru_assign_error = false
+    parse('X = 1', :on_assign_error) {thru_assign_error = true}
+    assert_equal false, thru_assign_error
+    parse("def m\n X = 1\nend", :on_assign_error) {thru_assign_error = true}
+    assert_equal true, thru_assign_error
+    thru_assign_error = false
+    parse("def m\n X, a = 1, 2\nend", :on_assign_error) {thru_assign_error = true}
+    assert_equal true, thru_assign_error
+  end
+
+  def test_assign_error_const_toplevel
+    thru_assign_error = false
+    parse('::X = 1', :on_assign_error) {thru_assign_error = true}
+    assert_equal false, thru_assign_error
+    parse("def m\n ::X = 1\nend", :on_assign_error) {thru_assign_error = true}
+    assert_equal true, thru_assign_error
+    thru_assign_error = false
+    parse("def m\n ::X, a = 1, 2\nend", :on_assign_error) {thru_assign_error = true}
+    assert_equal true, thru_assign_error
+  end
+
+  def test_bare_assoc_hash
+    thru_bare_assoc_hash = false
+    parse('x[a=>b]', :on_bare_assoc_hash) {thru_bare_assoc_hash = true}
+    assert_equal true, thru_bare_assoc_hash
+    thru_bare_assoc_hash = false
+    parse('x[1, a=>b]', :on_bare_assoc_hash) {thru_bare_assoc_hash = true}
+    assert_equal true, thru_bare_assoc_hash
+    thru_bare_assoc_hash = false
+    parse('x(a=>b)', :on_bare_assoc_hash) {thru_bare_assoc_hash = true}
+    assert_equal true, thru_bare_assoc_hash
+    thru_bare_assoc_hash = false
+    parse('x(1, a=>b)', :on_bare_assoc_hash) {thru_bare_assoc_hash = true}
+    assert_equal true, thru_bare_assoc_hash
+  end
+
+  def test_begin
+    thru_begin = false
+    parse('begin end', :on_begin) {thru_begin = true}
+    assert_equal true, thru_begin
+  end
+
+  %w"and or + - * / % ** | ^ & <=> > >= < <= == === != =~ !~ << >> && ||".each do |op|
+    define_method("test_binary(#{op})") do
+      thru_binary = false
+      parse("a #{op} b", :on_binary) {thru_binary = true}
+      assert_equal true, thru_binary
+    end
+  end
+
+  def test_blockarg
+    thru_blockarg = false
+    parse("def a(&b) end", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+    thru_blockarg = false
+    parse("def a(x, &b) end", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+
+    thru_blockarg = false
+    parse("proc{|&b|}", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+    thru_blockarg = false
+    parse("proc{|x, &b|}", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+    thru_blockarg = false
+    parse("proc{|&b;y|}", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+    thru_blockarg = false
+    parse("proc{|&b,x;y|}", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+
+    thru_blockarg = false
+    parse("proc do |&b| end", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+    thru_blockarg = false
+    parse("proc do |&b, x| end", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+    thru_blockarg = false
+    parse("proc do |&b;y| end", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+    thru_blockarg = false
+    parse("proc do |&b, x;y| end", :on_blockarg) {thru_blockarg = true}
+    assert_equal true, thru_blockarg
+  end
+
+  def test_block_var
+    thru_block_var = false
+    parse("proc{||}", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+    thru_block_var = false
+    parse("proc{| |}", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+    thru_block_var = false
+    parse("proc{|x|}", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+    thru_block_var = false
+    parse("proc{|;y|}", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+    thru_block_var = false
+    parse("proc{|x;y|}", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+
+    thru_block_var = false
+    parse("proc do || end", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+    thru_block_var = false
+    parse("proc do | | end", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+    thru_block_var = false
+    parse("proc do |x| end", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+    thru_block_var = false
+    parse("proc do |;y| end", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+    thru_block_var = false
+    parse("proc do |x;y| end", :on_block_var) {thru_block_var = true}
+    assert_equal true, thru_block_var
+  end
+
+  def test_block_var_add_block
+    # not used
+  end
+
+  def test_block_var_add_star
+    # not used
+  end
+
+  def test_bodystmt
+    thru_bodystmt = false
+    parse("class X\nend", :on_bodystmt) {thru_bodystmt = true}
+    assert_equal true, thru_bodystmt
+  end
+
+  def test_call
+    bug2233 = '[ruby-core:26165]'
+    tree = nil
+
+    thru_call = false
+    assert_nothing_raised {
+      tree = parse("self.foo", :on_call) {thru_call = true}
+    }
+    assert_equal true, thru_call
+    assert_equal "[call(ref(self),.,foo)]", tree
+    thru_call = false
+    assert_nothing_raised(bug2233) {
+      tree = parse("foo.()", :on_call) {thru_call = true}
+    }
+    assert_equal true, thru_call
+    assert_equal "[call(vcall(foo),.,call,[])]", tree
+  end
+
+  def test_excessed_comma
+    thru_excessed_comma = false
+    parse("proc{|x,|}", :on_excessed_comma) {thru_excessed_comma = true}
+    assert_equal true, thru_excessed_comma
+    thru_excessed_comma = false
+    parse("proc{|x,y,|}", :on_excessed_comma) {thru_excessed_comma = true}
+    assert_equal true, thru_excessed_comma
+
+    thru_excessed_comma = false
+    parse("proc do |x,| end", :on_excessed_comma) {thru_excessed_comma = true}
+    assert_equal true, thru_excessed_comma
+    thru_excessed_comma = false
+    parse("proc do |x,y,| end", :on_excessed_comma) {thru_excessed_comma = true}
+    assert_equal true, thru_excessed_comma
+  end
+
+  def test_heredoc
+    bug1921 = '[ruby-core:24855]'
+    thru_heredoc_beg = false
+    tree = parse("<""<EOS\nheredoc\nEOS\n", :on_heredoc_beg) {thru_heredoc_beg = true}
+    assert_equal true, thru_heredoc_beg
+    assert_match(/string_content\(\),heredoc\n/, tree, bug1921)
+    heredoc = nil
+    parse("<""<EOS\nheredoc1\nheredoc2\nEOS\n", :on_string_add) {|e, n, s| heredoc = s}
+    assert_equal("heredoc1\nheredoc2\n", heredoc, bug1921)
+    heredoc = nil
+    parse("<""<-EOS\nheredoc1\nheredoc2\n\tEOS\n", :on_string_add) {|e, n, s| heredoc = s}
+    assert_equal("heredoc1\nheredoc2\n", heredoc, bug1921)
+  end
+
+  def test_massign
+    thru_massign = false
+    parse("a, b = 1, 2", :on_massign) {thru_massign = true}
+    assert_equal true, thru_massign
+  end
+
+  def test_mlhs_add
+    thru_mlhs_add = false
+    parse("a, b = 1, 2", :on_mlhs_add) {thru_mlhs_add = true}
+    assert_equal true, thru_mlhs_add
+  end
+
+  def test_mlhs_add_star
+    bug2232 = '[ruby-core:26163]'
+    bug4364 = '[ruby-core:35078]'
+
+    thru_mlhs_add_star = false
+    tree = parse("a, *b = 1, 2", :on_mlhs_add_star) {thru_mlhs_add_star = true}
+    assert_equal true, thru_mlhs_add_star
+    assert_match(/mlhs_add_star\(mlhs_add\(mlhs_new\(\),a\),b\)/, tree)
+    thru_mlhs_add_star = false
+    tree = parse("a, *b, c = 1, 2", :on_mlhs_add_star) {thru_mlhs_add_star = true}
+    assert_equal true, thru_mlhs_add_star
+    assert_match(/mlhs_add\(mlhs_add_star\(mlhs_add\(mlhs_new\(\),a\),b\),mlhs_add\(mlhs_new\(\),c\)\)/, tree, bug2232)
+    thru_mlhs_add_star = false
+    tree = parse("a, *, c = 1, 2", :on_mlhs_add_star) {thru_mlhs_add_star = true}
+    assert_equal true, thru_mlhs_add_star
+    assert_match(/mlhs_add\(mlhs_add_star\(mlhs_add\(mlhs_new\(\),a\)\),mlhs_add\(mlhs_new\(\),c\)\)/, tree, bug4364)
+    thru_mlhs_add_star = false
+    tree = parse("*b, c = 1, 2", :on_mlhs_add_star) {thru_mlhs_add_star = true}
+    assert_equal true, thru_mlhs_add_star
+    assert_match(/mlhs_add\(mlhs_add_star\(mlhs_new\(\),b\),mlhs_add\(mlhs_new\(\),c\)\)/, tree, bug4364)
+    thru_mlhs_add_star = false
+    tree = parse("*, c = 1, 2", :on_mlhs_add_star) {thru_mlhs_add_star = true}
+    assert_equal true, thru_mlhs_add_star
+    assert_match(/mlhs_add\(mlhs_add_star\(mlhs_new\(\)\),mlhs_add\(mlhs_new\(\),c\)\)/, tree, bug4364)
+  end
+
+  def test_mlhs_new
+    thru_mlhs_new = false
+    parse("a, b = 1, 2", :on_mlhs_new) {thru_mlhs_new = true}
+    assert_equal true, thru_mlhs_new
+  end
+
+  def test_mlhs_paren
+    thru_mlhs_paren = false
+    parse("a, b = 1, 2", :on_mlhs_paren) {thru_mlhs_paren = true}
+    assert_equal false, thru_mlhs_paren
+    thru_mlhs_paren = false
+    parse("(a, b) = 1, 2", :on_mlhs_paren) {thru_mlhs_paren = true}
+    assert_equal true, thru_mlhs_paren
+  end
+
+  def test_brace_block
+    thru_brace_block = false
+    parse('proc {}', :on_brace_block) {thru_brace_block = true}
+    assert_equal true, thru_brace_block
+  end
+
+  def test_break
+    thru_break = false
+    parse('proc {break}', :on_break) {thru_break = true}
+    assert_equal true, thru_break
+  end
+
+  def test_case
+    thru_case = false
+    parse('case foo when true; end', :on_case) {thru_case = true}
+    assert_equal true, thru_case
+  end
+
+  def test_class
+    thru_class = false
+    parse('class Foo; end', :on_class) {thru_class = true}
+    assert_equal true, thru_class
+  end
+
+  def test_class_name_error
+    thru_class_name_error = false
+    parse('class foo; end', :on_class_name_error) {thru_class_name_error = true}
+    assert_equal true, thru_class_name_error
+  end
+
+  def test_command
+    thru_command = false
+    parse('foo a b', :on_command) {thru_command = true}
+    assert_equal true, thru_command
+  end
+
+  def test_command_call
+    thru_command_call = false
+    parse('foo.bar a, b', :on_command_call) {thru_command_call = true}
+    assert_equal true, thru_command_call
+  end
+
+  def test_const_ref
+    thru_const_ref = false
+    parse('class A;end', :on_const_ref) {thru_const_ref = true}
+    assert_equal true, thru_const_ref
+    thru_const_ref = false
+    parse('module A;end', :on_const_ref) {thru_const_ref = true}
+    assert_equal true, thru_const_ref
+  end
+
+  def test_const_path_field
+    thru_const_path_field = false
+    parse('foo::X = 1', :on_const_path_field) {thru_const_path_field = true}
+    assert_equal true, thru_const_path_field
+  end
+
+  def test_const_path_ref
+    thru_const_path_ref = false
+    parse('foo::X', :on_const_path_ref) {thru_const_path_ref = true}
+    assert_equal true, thru_const_path_ref
+  end
+
+  def test_def
+    thru_def = false
+    parse('def foo; end', :on_def) {
+      thru_def = true
+    }
+    assert_equal true, thru_def
+    assert_equal '[def(foo,[],bodystmt([void()]))]', parse('def foo ;end')
+  end
+
+  def test_defined
+    thru_defined = false
+    parse('defined?(x)', :on_defined) {thru_defined = true}
+    assert_equal true, thru_defined
+  end
+
+  def test_defs
+    thru_defs = false
+    parse('def foo.bar; end', :on_defs) {thru_defs = true}
+    assert_equal true, thru_defs
+  end
+
+  def test_do_block
+    thru_do_block = false
+    parse('proc do end', :on_do_block) {thru_do_block = true}
+    assert_equal true, thru_do_block
+  end
+
+  def test_dot2
+    thru_dot2 = false
+    parse('a..b', :on_dot2) {thru_dot2 = true}
+    assert_equal true, thru_dot2
+  end
+
+  def test_dot3
+    thru_dot3 = false
+    parse('a...b', :on_dot3) {thru_dot3 = true}
+    assert_equal true, thru_dot3
+  end
+
+  def test_dyna_symbol
+    thru_dyna_symbol = false
+    parse(':"#{foo}"', :on_dyna_symbol) {thru_dyna_symbol = true}
+    assert_equal true, thru_dyna_symbol
+
+    thru_dyna_symbol = false
+    parse('{"#{foo}": 1}', :on_dyna_symbol) {thru_dyna_symbol = true}
+    assert_equal true, thru_dyna_symbol
+  end
+
+  def test_else
+    thru_else = false
+    parse('if foo; bar else zot end', :on_else) {thru_else = true}
+    assert_equal true, thru_else
+  end
+
+  def test_elsif
+    thru_elsif = false
+    parse('if foo; bar elsif qux; zot end', :on_elsif) {thru_elsif = true}
+    assert_equal true, thru_elsif
+  end
+
+  def test_ensure
+    thru_ensure = false
+    parse('begin foo ensure bar end', :on_ensure) {thru_ensure = true}
+    assert_equal true, thru_ensure
+  end
+
+  def test_fcall
+    thru_fcall = false
+    parse('foo()', :on_fcall) {thru_fcall = true}
+    assert_equal true, thru_fcall
+  end
+
+  def test_field
+    thru_field = false
+    parse('foo.x = 1', :on_field) {thru_field = true}
+    assert_equal true, thru_field
+  end
+
+  def test_for
+    thru_for = false
+    parse('for i in foo; end', :on_for) {thru_for = true}
+    assert_equal true, thru_for
+  end
+
+  def test_hash
+    thru_hash = false
+    parse('{1=>2}', :on_hash) {thru_hash = true}
+    assert_equal true, thru_hash
+    thru_hash = false
+    parse('{a: 2}', :on_hash) {thru_hash = true}
+    assert_equal true, thru_hash
+  end
+
+  def test_if
+    thru_if = false
+    parse('if false; end', :on_if) {thru_if = true}
+    assert_equal true, thru_if
+  end
+
+  def test_if_mod
+    thru_if_mod = false
+    parse('nil if nil', :on_if_mod) {thru_if_mod = true}
+    assert_equal true, thru_if_mod
+  end
+
+  def test_ifop
+    thru_ifop = false
+    parse('a ? b : c', :on_ifop) {thru_ifop = true}
+    assert_equal true, thru_ifop
+  end
+
+  def test_lambda
+    thru_lambda = false
+    parse('->{}', :on_lambda) {thru_lambda = true}
+    assert_equal true, thru_lambda
+  end
+
+  def test_magic_comment
+    thru_magic_comment = false
+    parse('# -*- bug-5753: ruby-dev:44984 -*-', :on_magic_comment) {|*x|thru_magic_comment = x}
+    assert_equal [:on_magic_comment, "bug_5753", "ruby-dev:44984"], thru_magic_comment
+  end
+
+  def test_method_add_block
+    thru_method_add_block = false
+    parse('a {}', :on_method_add_block) {thru_method_add_block = true}
+    assert_equal true, thru_method_add_block
+    thru_method_add_block = false
+    parse('a do end', :on_method_add_block) {thru_method_add_block = true}
+    assert_equal true, thru_method_add_block
+  end
+
+  def test_method_add_arg
+    thru_method_add_arg = false
+    parse('a()', :on_method_add_arg) {thru_method_add_arg = true}
+    assert_equal true, thru_method_add_arg
+    thru_method_add_arg = false
+    parse('a {}', :on_method_add_arg) {thru_method_add_arg = true}
+    assert_equal true, thru_method_add_arg
+    thru_method_add_arg = false
+    parse('a.b(1)', :on_method_add_arg) {thru_method_add_arg = true}
+    assert_equal true, thru_method_add_arg
+    thru_method_add_arg = false
+    parse('a::b(1)', :on_method_add_arg) {thru_method_add_arg = true}
+    assert_equal true, thru_method_add_arg
+  end
+
+  def test_module
+    thru_module = false
+    parse('module A; end', :on_module) {thru_module = true}
+    assert_equal true, thru_module
+  end
+
+  def test_mrhs_add
+    thru_mrhs_add = false
+    parse('a = a, b', :on_mrhs_add) {thru_mrhs_add = true}
+    assert_equal true, thru_mrhs_add
+  end
+
+  def test_mrhs_add_star
+    thru_mrhs_add_star = false
+    parse('a = a, *b', :on_mrhs_add_star) {thru_mrhs_add_star = true}
+    assert_equal true, thru_mrhs_add_star
+  end
+
+  def test_mrhs_new
+    thru_mrhs_new = false
+    parse('a = *a', :on_mrhs_new) {thru_mrhs_new = true}
+    assert_equal true, thru_mrhs_new
+  end
+
+  def test_mrhs_new_from_args
+    thru_mrhs_new_from_args = false
+    parse('a = a, b', :on_mrhs_new_from_args) {thru_mrhs_new_from_args = true}
+    assert_equal true, thru_mrhs_new_from_args
+  end
+
+  def test_next
+    thru_next = false
+    parse('a {next}', :on_next) {thru_next = true}
+    assert_equal true, thru_next
+  end
+
+  def test_opassign
+    thru_opassign = false
+    parse('a += b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a -= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a *= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a /= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a %= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a **= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a &= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a |= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a <<= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a >>= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a &&= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a ||= b', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+    thru_opassign = false
+    parse('a::X ||= c 1', :on_opassign) {thru_opassign = true}
+    assert_equal true, thru_opassign
+  end
+
+  def test_opassign_error
+    thru_opassign = []
+    events = [:on_opassign]
+    parse('$~ ||= 1', events) {|a,*b|
+      thru_opassign << a
+    }
+    assert_equal events, thru_opassign
+  end
+
+  def test_param_error
+    thru_param_error = false
+    parse('def foo(A) end', :on_param_error) {thru_param_error = true}
+    assert_equal true, thru_param_error
+    thru_param_error = false
+    parse('def foo($a) end', :on_param_error) {thru_param_error = true}
+    assert_equal true, thru_param_error
+    thru_param_error = false
+    parse('def foo(@a) end', :on_param_error) {thru_param_error = true}
+    assert_equal true, thru_param_error
+    thru_param_error = false
+    parse('def foo(@@a) end', :on_param_error) {thru_param_error = true}
+    assert_equal true, thru_param_error
+  end
+
+  def test_params
+    arg = nil
+    thru_params = false
+    parse('a {||}', :on_params) {|_, *v| thru_params = true; arg = v}
+    assert_equal true, thru_params
+    assert_equal [nil, nil, nil, nil, nil, nil, nil], arg
+    thru_params = false
+    parse('a {|x|}', :on_params) {|_, *v| thru_params = true; arg = v}
+    assert_equal true, thru_params
+    assert_equal [["x"], nil, nil, nil, nil, nil, nil], arg
+    thru_params = false
+    parse('a {|*x|}', :on_params) {|_, *v| thru_params = true; arg = v}
+    assert_equal true, thru_params
+    assert_equal [nil, nil, "*x", nil, nil, nil, nil], arg
+    thru_params = false
+    parse('a {|x: 1|}', :on_params) {|_, *v| thru_params = true; arg = v}
+    assert_equal true, thru_params
+    assert_equal [nil, nil, nil, nil, [["x:", "1"]], nil, nil], arg
+    thru_params = false
+    parse('a {|x:|}', :on_params) {|_, *v| thru_params = true; arg = v}
+    assert_equal true, thru_params
+    assert_equal [nil, nil, nil, nil, [["x:", false]], nil, nil], arg
+    thru_params = false
+    parse('a {|**x|}', :on_params) {|_, *v| thru_params = true; arg = v}
+    assert_equal true, thru_params
+    assert_equal [nil, nil, nil, nil, nil, "x", nil], arg
+  end
+
+  def test_paren
+    thru_paren = false
+    parse('()', :on_paren) {thru_paren = true}
+    assert_equal true, thru_paren
+  end
+
+  def test_parse_error
+    thru_parse_error = false
+    parse('<>', :on_parse_error) {thru_parse_error = true}
+    assert_equal true, thru_parse_error
+  end
+
+  def test_qwords_add
+    thru_qwords_add = false
+    parse('%w[a]', :on_qwords_add) {thru_qwords_add = true}
+    assert_equal true, thru_qwords_add
+  end
+
+  def test_qsymbols_add
+    thru_qsymbols_add = false
+    parse('%i[a]', :on_qsymbols_add) {thru_qsymbols_add = true}
+    assert_equal true, thru_qsymbols_add
+  end
+
+  def test_symbols_add
+    thru_symbols_add = false
+    parse('%I[a]', :on_symbols_add) {thru_symbols_add = true}
+    assert_equal true, thru_symbols_add
+  end
+
+  def test_qwords_new
+    thru_qwords_new = false
+    parse('%w[]', :on_qwords_new) {thru_qwords_new = true}
+    assert_equal true, thru_qwords_new
+  end
+
+  def test_qsymbols_new
+    thru_qsymbols_new = false
+    parse('%i[]', :on_qsymbols_new) {thru_qsymbols_new = true}
+    assert_equal true, thru_qsymbols_new
+  end
+
+  def test_symbols_new
+    thru_symbols_new = false
+    parse('%I[]', :on_symbols_new) {thru_symbols_new = true}
+    assert_equal true, thru_symbols_new
+  end
+
+  def test_redo
+    thru_redo = false
+    parse('redo', :on_redo) {thru_redo = true}
+    assert_equal true, thru_redo
+  end
+
+  def test_regexp_add
+    thru_regexp_add = false
+    parse('/foo/', :on_regexp_add) {thru_regexp_add = true}
+    assert_equal true, thru_regexp_add
+  end
+
+  def test_regexp_literal
+    thru_regexp_literal = false
+    parse('//', :on_regexp_literal) {thru_regexp_literal = true}
+    assert_equal true, thru_regexp_literal
+  end
+
+  def test_regexp_new
+    thru_regexp_new = false
+    parse('//', :on_regexp_new) {thru_regexp_new = true}
+    assert_equal true, thru_regexp_new
+  end
+
+  def test_rescue
+    thru_rescue = false
+    parsed = parse('begin; 1; rescue => e; 2; end', :on_rescue) {thru_rescue = true}
+    assert_equal true, thru_rescue
+    assert_match(/1.*rescue/, parsed)
+    assert_match(/rescue\(,var_field\(e\),\[2\]\)/, parsed)
+  end
+
+  def test_rescue_class
+    thru_rescue = false
+    parsed = parse('begin; 1; rescue RuntimeError => e; 2; end', :on_rescue) {thru_rescue = true}
+    assert_equal true, thru_rescue
+    assert_match(/1.*rescue/, parsed)
+    assert_match(/rescue\(\[ref\(RuntimeError\)\],var_field\(e\),\[2\]\)/, parsed)
+  end
+
+  def test_rescue_mod
+    thru_rescue_mod = false
+    parsed = parse('1 rescue 2', :on_rescue_mod) {thru_rescue_mod = true}
+    assert_equal true, thru_rescue_mod
+    bug4716 = '[ruby-core:36248]'
+    assert_equal "[rescue_mod(1,2)]", parsed, bug4716
+  end
+
+  def test_rest_param
+    thru_rest_param = false
+    parse('def a(*) end', :on_rest_param) {thru_rest_param = true}
+    assert_equal true, thru_rest_param
+    thru_rest_param = false
+    parse('def a(*x) end', :on_rest_param) {thru_rest_param = true}
+    assert_equal true, thru_rest_param
+  end
+
+  def test_retry
+    thru_retry = false
+    parse('retry', :on_retry) {thru_retry = true}
+    assert_equal true, thru_retry
+  end
+
+  def test_return
+    thru_return = false
+    parse('return a', :on_return) {thru_return = true}
+    assert_equal true, thru_return
+  end
+
+  def test_return0
+    thru_return0 = false
+    parse('return', :on_return0) {thru_return0 = true}
+    assert_equal true, thru_return0
+  end
+
+  def test_sclass
+    thru_sclass = false
+    parse('class << a; end', :on_sclass) {thru_sclass = true}
+    assert_equal true, thru_sclass
+  end
+
+  def test_string_add
+    thru_string_add = false
+    parse('"aa"', :on_string_add) {thru_string_add = true}
+    assert_equal true, thru_string_add
+  end
+
+  def test_string_concat
+    thru_string_concat = false
+    parse('"a" "b"', :on_string_concat) {thru_string_concat = true}
+    assert_equal true, thru_string_concat
+  end
+
+  def test_string_content
+    thru_string_content = false
+    parse('""', :on_string_content) {thru_string_content = true}
+    assert_equal true, thru_string_content
+    thru_string_content = false
+    parse('"a"', :on_string_content) {thru_string_content = true}
+    assert_equal true, thru_string_content
+    thru_string_content = false
+    parse('%[a]', :on_string_content) {thru_string_content = true}
+    assert_equal true, thru_string_content
+    thru_string_content = false
+    parse('\'a\'', :on_string_content) {thru_string_content = true}
+    assert_equal true, thru_string_content
+    thru_string_content = false
+    parse('%<a>', :on_string_content) {thru_string_content = true}
+    assert_equal true, thru_string_content
+    thru_string_content = false
+    parse('%!a!', :on_string_content) {thru_string_content = true}
+    assert_equal true, thru_string_content
+    thru_string_content = false
+    parse('%q!a!', :on_string_content) {thru_string_content = true}
+    assert_equal true, thru_string_content
+    thru_string_content = false
+    parse('%Q!a!', :on_string_content) {thru_string_content = true}
+    assert_equal true, thru_string_content
+  end
+
+  def test_string_dvar
+    thru_string_dvar = false
+    parse('"#$a"', :on_string_dvar) {thru_string_dvar = true}
+    assert_equal true, thru_string_dvar
+    thru_string_dvar = false
+    parse('\'#$a\'', :on_string_dvar) {thru_string_dvar = true}
+    assert_equal false, thru_string_dvar
+    thru_string_dvar = false
+    parse('"#@a"', :on_string_dvar) {thru_string_dvar = true}
+    assert_equal true, thru_string_dvar
+    thru_string_dvar = false
+    parse('\'#@a\'', :on_string_dvar) {thru_string_dvar = true}
+    assert_equal false, thru_string_dvar
+    thru_string_dvar = false
+    parse('"#@@a"', :on_string_dvar) {thru_string_dvar = true}
+    assert_equal true, thru_string_dvar
+    thru_string_dvar = false
+    parse('\'#@@a\'', :on_string_dvar) {thru_string_dvar = true}
+    assert_equal false, thru_string_dvar
+    thru_string_dvar = false
+    parse('"#$1"', :on_string_dvar) {thru_string_dvar = true}
+    assert_equal true, thru_string_dvar
+    thru_string_dvar = false
+    parse('\'#$1\'', :on_string_dvar) {thru_string_dvar = true}
+    assert_equal false, thru_string_dvar
+  end
+
+  def test_string_embexpr
+    thru_string_embexpr = false
+    parse('"#{}"', :on_string_embexpr) {thru_string_embexpr = true}
+    assert_equal true, thru_string_embexpr
+    thru_string_embexpr = false
+    parse('\'#{}\'', :on_string_embexpr) {thru_string_embexpr = true}
+    assert_equal false, thru_string_embexpr
+  end
+
+  def test_string_literal
+    thru_string_literal = false
+    parse('""', :on_string_literal) {thru_string_literal = true}
+    assert_equal true, thru_string_literal
+  end
+
+  def test_super
+    thru_super = false
+    parse('super()', :on_super) {thru_super = true}
+    assert_equal true, thru_super
+  end
+
+  def test_symbol
+    thru_symbol = false
+    parse(':a', :on_symbol) {thru_symbol = true}
+    assert_equal true, thru_symbol
+    thru_symbol = false
+    parse(':$a', :on_symbol) {thru_symbol = true}
+    assert_equal true, thru_symbol
+    thru_symbol = false
+    parse(':@a', :on_symbol) {thru_symbol = true}
+    assert_equal true, thru_symbol
+    thru_symbol = false
+    parse(':@@a', :on_symbol) {thru_symbol = true}
+    assert_equal true, thru_symbol
+    thru_symbol = false
+    parse(':==', :on_symbol) {thru_symbol = true}
+    assert_equal true, thru_symbol
+  end
+
+  def test_symbol_literal
+    thru_symbol_literal = false
+    parse(':a', :on_symbol_literal) {thru_symbol_literal = true}
+    assert_equal true, thru_symbol_literal
+  end
+
+  def test_top_const_field
+    thru_top_const_field = false
+    parse('::A=1', :on_top_const_field) {thru_top_const_field = true}
+    assert_equal true, thru_top_const_field
+  end
+
+  def test_top_const_ref
+    thru_top_const_ref = false
+    parse('::A', :on_top_const_ref) {thru_top_const_ref = true}
+    assert_equal true, thru_top_const_ref
+  end
+
+  def test_unary
+    thru_unary = false
+    parse('not a 1, 2', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('not (a)', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('!a', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('-10', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('-10*2', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('-10.1', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('-10.1*2', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('-a', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('+a', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('~a', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+    thru_unary = false
+    parse('not()', :on_unary) {thru_unary = true}
+    assert_equal true, thru_unary
+  end
+
+  def test_undef
+    thru_undef = false
+    parse('undef a', :on_undef) {thru_undef = true}
+    assert_equal true, thru_undef
+    thru_undef = false
+    parse('undef <=>', :on_undef) {thru_undef = true}
+    assert_equal true, thru_undef
+    thru_undef = false
+    parse('undef a, b', :on_undef) {thru_undef = true}
+    assert_equal true, thru_undef
+  end
+
+  def test_unless
+    thru_unless = false
+    parse('unless a; end', :on_unless) {thru_unless = true}
+    assert_equal true, thru_unless
+  end
+
+  def test_unless_mod
+    thru_unless_mod = false
+    parse('nil unless a', :on_unless_mod) {thru_unless_mod = true}
+    assert_equal true, thru_unless_mod
+  end
+
+  def test_until
+    thru_until = false
+    parse('until a; end', :on_until) {thru_until = true}
+    assert_equal true, thru_until
+  end
+
+  def test_until_mod
+    thru_until_mod = false
+    parse('nil until a', :on_until_mod) {thru_until_mod = true}
+    assert_equal true, thru_until_mod
+  end
+
+  def test_var_field
+    thru_var_field = false
+    parse('a = 1', :on_var_field) {thru_var_field = true}
+    assert_equal true, thru_var_field
+    thru_var_field = false
+    parse('a += 1', :on_var_field) {thru_var_field = true}
+    assert_equal true, thru_var_field
+  end
+
+  def test_when
+    thru_when = false
+    parse('case a when b; end', :on_when) {thru_when = true}
+    assert_equal true, thru_when
+    thru_when = false
+    parse('case when a; end', :on_when) {thru_when = true}
+    assert_equal true, thru_when
+  end
+
+  def test_while
+    thru_while = false
+    parse('while a; end', :on_while) {thru_while = true}
+    assert_equal true, thru_while
+  end
+
+  def test_while_mod
+    thru_while_mod = false
+    parse('nil while a', :on_while_mod) {thru_while_mod = true}
+    assert_equal true, thru_while_mod
+  end
+
+  def test_word_add
+    thru_word_add = false
+    parse('%W[a]', :on_word_add) {thru_word_add = true}
+    assert_equal true, thru_word_add
+  end
+
+  def test_word_new
+    thru_word_new = false
+    parse('%W[a]', :on_word_new) {thru_word_new = true}
+    assert_equal true, thru_word_new
+  end
+
+  def test_words_add
+    thru_words_add = false
+    parse('%W[a]', :on_words_add) {thru_words_add = true}
+    assert_equal true, thru_words_add
+  end
+
+  def test_words_new
+    thru_words_new = false
+    parse('%W[]', :on_words_new) {thru_words_new = true}
+    assert_equal true, thru_words_new
+  end
+
+  def test_xstring_add
+    thru_xstring_add = false
+    parse('`x`', :on_xstring_add) {thru_xstring_add = true}
+    assert_equal true, thru_xstring_add
+  end
+
+  def test_xstring_literal
+    thru_xstring_literal = false
+    parse('``', :on_xstring_literal) {thru_xstring_literal = true}
+    assert_equal true, thru_xstring_literal
+  end
+
+  def test_xstring_new
+    thru_xstring_new = false
+    parse('``', :on_xstring_new) {thru_xstring_new = true}
+    assert_equal true, thru_xstring_new
+  end
+
+  def test_yield
+    thru_yield = false
+    parse('yield a', :on_yield) {thru_yield = true}
+    assert_equal true, thru_yield
+  end
+
+  def test_yield0
+    thru_yield0 = false
+    parse('yield', :on_yield0) {thru_yield0 = true}
+    assert_equal true, thru_yield0
+  end
+
+  def test_zsuper
+    thru_zsuper = false
+    parse('super', :on_zsuper) {thru_zsuper = true}
+    assert_equal true, thru_zsuper
+  end
+
+  def test_local_variables
+    cmd = 'command(w,[regexp_literal(regexp_add(regexp_new(),25 # ),/)])'
+    div = 'binary(ref(w),/,25)'
+    bug1939 = '[ruby-core:24923]'
+
+    assert_equal("[#{cmd}]", parse('w /25 # /'), bug1939)
+    assert_equal("[assign(var_field(w),1),#{div}]", parse("w = 1; w /25 # /"), bug1939)
+    assert_equal("[fcall(p,[],&block([w],[#{div}]))]", parse("p{|w|w /25 # /\n}"), bug1939)
+    assert_equal("[def(p,[w],bodystmt([#{div}]))]", parse("def p(w)\nw /25 # /\nend"), bug1939)
+  end
+
+  def test_block_variables
+    assert_equal("[fcall(proc,[],&block([],[void()]))]", parse("proc{|;y|}"))
+    if defined?(Process::RLIMIT_AS)
+      assert_in_out_err(["-I#{File.dirname(__FILE__)}", "-rdummyparser"],
+                        'Process.setrlimit(Process::RLIMIT_AS,100*1024*1024); puts DummyParser.new("proc{|;y|!y}").parse',
+                        ["[fcall(proc,[],&block([],[unary(!,ref(y))]))]"], [], '[ruby-dev:39423]')
+    end
+  end
+
+  def test_unterminated_regexp
+    assert_equal("unterminated regexp meets end of file", compile_error('/'))
+  end
+
+  def test_invalid_instance_variable_name
+    assert_equal("`@1' is not allowed as an instance variable name", compile_error('@1'))
+    assert_equal("`@%' is not allowed as an instance variable name", compile_error('@%'))
+  end
+
+  def test_invalid_class_variable_name
+    assert_equal("`@@1' is not allowed as a class variable name", compile_error('@@1'))
+    assert_equal("`@@%' is not allowed as a class variable name", compile_error('@@%'))
+  end
+
+  def test_invalid_global_variable_name
+    assert_equal("`$%' is not allowed as a global variable name", compile_error('$%'))
+  end
+end if ripper_test
diff --git a/jni/ruby/test/ripper/test_ripper.rb b/jni/ruby/test/ripper/test_ripper.rb
new file mode 100644
index 0000000..1544e56
--- /dev/null
+++ b/jni/ruby/test/ripper/test_ripper.rb
@@ -0,0 +1,63 @@
+begin
+  require 'ripper'
+  require 'test/unit'
+  ripper_test = true
+  module TestRipper; end
+rescue LoadError
+end
+
+class TestRipper::Ripper < Test::Unit::TestCase
+
+  def setup
+    @ripper = Ripper.new '1 + 1'
+  end
+
+  def test_column
+    assert_nil @ripper.column
+  end
+
+  def test_encoding
+    assert_equal Encoding::UTF_8, @ripper.encoding
+    ripper = Ripper.new('# coding: iso-8859-15')
+    ripper.parse
+    assert_equal Encoding::ISO_8859_15, ripper.encoding
+    ripper = Ripper.new('# -*- coding: iso-8859-15 -*-')
+    ripper.parse
+    assert_equal Encoding::ISO_8859_15, ripper.encoding
+  end
+
+  def test_end_seen_eh
+    @ripper.parse
+    assert_not_predicate @ripper, :end_seen?
+    ripper = Ripper.new('__END__')
+    ripper.parse
+    assert_predicate ripper, :end_seen?
+  end
+
+  def test_filename
+    assert_equal '(ripper)', @ripper.filename
+    filename = "ripper"
+    ripper = Ripper.new("", filename)
+    filename.clear
+    assert_equal "ripper", ripper.filename
+  end
+
+  def test_lineno
+    assert_nil @ripper.lineno
+  end
+
+  def test_parse
+    assert_nil @ripper.parse
+  end
+
+  def test_yydebug
+    assert_not_predicate @ripper, :yydebug
+  end
+
+  def test_yydebug_equals
+    @ripper.yydebug = true
+
+    assert_predicate @ripper, :yydebug
+  end
+
+end if ripper_test
diff --git a/jni/ruby/test/ripper/test_scanner_events.rb b/jni/ruby/test/ripper/test_scanner_events.rb
new file mode 100644
index 0000000..e57300e
--- /dev/null
+++ b/jni/ruby/test/ripper/test_scanner_events.rb
@@ -0,0 +1,909 @@
+#
+# test_scanner_events.rb
+#
+begin
+  require 'ripper'
+  require 'test/unit'
+  ripper_test = true
+  module TestRipper; end
+rescue LoadError
+end
+
+class TestRipper::ScannerEvents < Test::Unit::TestCase
+
+  def test_event_coverage
+    dispatched = Ripper::SCANNER_EVENTS.map {|event,_| event }
+    dispatched.each do |e|
+      assert_equal true, respond_to?("test_#{e}", true), "event not tested: #{e}"
+    end
+  end
+
+  def scan(target, str)
+    sym = "on_#{target}".intern
+    Ripper.lex(str).select {|_1,type,_2| type == sym }.map {|_1,_2,tok| tok }
+  end
+
+  def test_tokenize
+    assert_equal [],
+                 Ripper.tokenize('')
+    assert_equal ['a'],
+                 Ripper.tokenize('a')
+    assert_equal ['1'],
+                 Ripper.tokenize('1')
+    assert_equal ['1', ';', 'def', ' ', 'm', '(', 'arg', ')', 'end'],
+                 Ripper.tokenize("1;def m(arg)end")
+    assert_equal ['print', '(', '<<''EOS', ')', "\n", "heredoc\n", "EOS\n"],
+                 Ripper.tokenize("print(<<""EOS)\nheredoc\nEOS\n")
+    assert_equal ['print', '(', ' ', '<<''EOS', ')', "\n", "heredoc\n", "EOS\n"],
+                 Ripper.tokenize("print( <<""EOS)\nheredoc\nEOS\n")
+    assert_equal ["\#\n", "\n", "\#\n", "\n", "nil", "\n"],
+                 Ripper.tokenize("\#\n\n\#\n\nnil\n")
+    assert_equal ["1", "  ", ".", "foo", "\n"],
+                 Ripper.tokenize("1  .foo\n")
+    assert_equal ["1", "\n", "  ", ".", "foo", "\n"],
+                 Ripper.tokenize("1\n  .foo\n")
+  end
+
+  def test_lex
+    assert_equal [],
+                 Ripper.lex('')
+    assert_equal [[[1,0], :on_ident, "a"]],
+                 Ripper.lex('a')
+    assert_equal [[[1, 0], :on_kw, "nil"]],
+                 Ripper.lex("nil")
+    assert_equal [[[1, 0], :on_kw, "def"],
+                  [[1, 3], :on_sp, " "],
+                  [[1, 4], :on_ident, "m"],
+                  [[1, 5], :on_lparen, "("],
+                  [[1, 6], :on_ident, "a"],
+                  [[1, 7], :on_rparen, ")"],
+                  [[1, 8], :on_kw, "end"]],
+                 Ripper.lex("def m(a)end")
+    assert_equal [[[1, 0], :on_int, "1"],
+                  [[1, 1], :on_nl, "\n"],
+                  [[2, 0], :on_int, "2"],
+                  [[2, 1], :on_nl, "\n"],
+                  [[3, 0], :on_int, "3"]],
+                 Ripper.lex("1\n2\n3")
+    assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"],
+                  [[1, 5], :on_nl, "\n"],
+                  [[2, 0], :on_tstring_content, "heredoc\n"],
+                  [[3, 0], :on_heredoc_end, "EOS"]],
+                 Ripper.lex("<<""EOS\nheredoc\nEOS")
+    assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"],
+                  [[1, 5], :on_nl, "\n"],
+                  [[2, 0], :on_heredoc_end, "EOS"]],
+                 Ripper.lex("<<""EOS\nEOS"),
+                 "bug#4543"
+    assert_equal [[[1, 0], :on_regexp_beg, "/"],
+                  [[1, 1], :on_tstring_content, "foo\nbar"],
+                  [[2, 3], :on_regexp_end, "/"]],
+                 Ripper.lex("/foo\nbar/")
+    assert_equal [[[1, 0], :on_regexp_beg, "/"],
+                  [[1, 1], :on_tstring_content, "foo\n\u3020"],
+                  [[2, 3], :on_regexp_end, "/"]],
+                 Ripper.lex("/foo\n\u3020/")
+    assert_equal [[[1, 0], :on_tstring_beg, "'"],
+                  [[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0"],
+                  [[2, 3], :on_tstring_end, "'"]],
+                 Ripper.lex("'foo\n\xe3\x80\xa0'")
+    assert_equal [[[1, 0], :on_tstring_beg, "'"],
+                  [[1, 1], :on_tstring_content, "\u3042\n\u3044"],
+                  [[2, 3], :on_tstring_end, "'"]],
+                 Ripper.lex("'\u3042\n\u3044'")
+    assert_equal [[[1, 0], :on_rational, "1r"],
+                  [[1, 2], :on_nl, "\n"],
+                  [[2, 0], :on_imaginary, "2i"],
+                  [[2, 2], :on_nl, "\n"],
+                  [[3, 0], :on_imaginary, "3ri"],
+                  [[3, 3], :on_nl, "\n"],
+                  [[4, 0], :on_rational, "4.2r"],
+                  [[4, 4], :on_nl, "\n"],
+                  [[5, 0], :on_imaginary, "5.6ri"],
+                 ],
+                 Ripper.lex("1r\n2i\n3ri\n4.2r\n5.6ri")
+  end
+
+  def test_location
+    assert_location ""
+    assert_location " "
+    assert_location ":"
+    assert_location "\n"
+    assert_location "\r\n"
+    assert_location "\n\n\n\n\n\r\n\n\n"
+    assert_location "\n;\n;\n;\n;\n"
+    assert_location "nil"
+    assert_location "@ivar"
+    assert_location "1;2;3"
+    assert_location "1\n2\n3"
+    assert_location "1\n2\n3\n"
+    assert_location "def m(a) nil end"
+    assert_location "if true then false else nil end"
+    assert_location "BEGIN{print nil}"
+    assert_location "%w(a b\nc\r\nd \ne )"
+    assert_location %Q["a\nb\r\nc"]
+    assert_location "print(<<""EOS)\nheredoc\nEOS\n"
+    assert_location "print(<<-\"EOS\")\nheredoc\n     EOS\n"
+  end
+
+  def assert_location(src)
+    buf = ''
+    Ripper.lex(src).each do |pos, type, tok|
+      line, col = *pos
+      assert_equal buf.count("\n") + 1, line,
+          "wrong lineno: #{tok.inspect} (#{type}) [#{line}:#{col}]"
+      assert_equal buf.sub(/\A.*\n/m, '').size, col,
+          "wrong column: #{tok.inspect} (#{type}) [#{line}:#{col}]"
+      buf << tok
+    end
+    assert_equal src, buf
+  end
+
+  def test_backref
+    assert_equal ["$`", "$&", "$'", '$1', '$2', '$3'],
+                 scan('backref', %q[m($~, $`, $&, $', $1, $2, $3)])
+  end
+
+  def test_backtick
+    assert_equal ["`"],
+                 scan('backtick', %q[p `make all`])
+  end
+
+  def test_comma
+    assert_equal [','] * 6,
+                 scan('comma', %q[ m(0,1,2,3,4,5,6) ])
+    assert_equal [],
+                 scan('comma', %q[".,.,.,.,.,.,.."])
+    assert_equal [],
+                 scan('comma', "<<""EOS\n,,,,,,,,,,\nEOS")
+  end
+
+  def test_period
+    assert_equal [],
+                 scan('period', '')
+    assert_equal ['.'],
+                 scan('period', 'a.b')
+    assert_equal ['.'],
+                 scan('period', 'Object.new')
+    assert_equal [],
+                 scan('period', '"."')
+    assert_equal [],
+                 scan('period', '1..2')
+    assert_equal [],
+                 scan('period', '1...3')
+  end
+
+  def test_const
+    assert_equal ['CONST'],
+                 scan('const', 'CONST')
+    assert_equal ['C'],
+                 scan('const', 'C')
+    assert_equal ['CONST_A'],
+                 scan('const', 'CONST_A')
+    assert_equal ['Const', 'Const2', 'Const3'],
+                 scan('const', 'Const; Const2; Const3')
+    assert_equal ['Const'],
+                 scan('const', 'Const(a)')
+    assert_equal ['M', 'A', 'A2'],
+                 scan('const', 'M(A,A2)')
+    assert_equal [],
+                 scan('const', '')
+    assert_equal [],
+                 scan('const', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_cvar
+    assert_equal [],
+                 scan('cvar', '')
+    assert_equal ['@@cvar'],
+                 scan('cvar', '@@cvar')
+    assert_equal ['@@__cvar__'],
+                 scan('cvar', '@@__cvar__')
+    assert_equal ['@@CVAR'],
+                 scan('cvar', '@@CVAR')
+    assert_equal ['@@cvar'],
+                 scan('cvar', '   @@cvar#comment')
+    assert_equal ['@@cvar'],
+                 scan('cvar', ':@@cvar')
+    assert_equal ['@@cvar'],
+                 scan('cvar', 'm(lvar, @ivar, @@cvar, $gvar)')
+    assert_equal [],
+                 scan('cvar', '"@@cvar"')
+  end
+
+  def test_embexpr_beg
+    assert_equal [],
+                 scan('embexpr_beg', '')
+    assert_equal ['#{'],
+                 scan('embexpr_beg', '"#{expr}"')
+    assert_equal [],
+                 scan('embexpr_beg', '%q[#{expr}]')
+    assert_equal ['#{'],
+                 scan('embexpr_beg', '%Q[#{expr}]')
+    assert_equal ['#{'],
+                 scan('embexpr_beg', "m(<<""EOS)\n\#{expr}\nEOS")
+  end
+
+  def test_embexpr_end
+    assert_equal [],
+                 scan('embexpr_end', '')
+    assert_equal ['}'],
+                 scan('embexpr_end', '"#{expr}"')
+    assert_equal [],
+                 scan('embexpr_end', '%q[#{expr}]')
+    assert_equal ['}'],
+                 scan('embexpr_end', '%Q[#{expr}]')
+    assert_equal ['}'],
+                 scan('embexpr_end', "m(<<""EOS)\n\#{expr}\nEOS")
+  end
+
+  def test_embvar
+    assert_equal [],
+                 scan('embvar', '')
+    assert_equal ['#'],
+                 scan('embvar', '"#$gvar"')
+    assert_equal ['#'],
+                 scan('embvar', '"#@ivar"')
+    assert_equal ['#'],
+                 scan('embvar', '"#@@cvar"')
+    assert_equal [],
+                 scan('embvar', '"#lvar"')
+    assert_equal [],
+                 scan('embvar', '"#"')
+    assert_equal [],
+                 scan('embvar', '"\#$gvar"')
+    assert_equal [],
+                 scan('embvar', '"\#@ivar"')
+    assert_equal [],
+                 scan('embvar', '%q[#@ivar]')
+    assert_equal ['#'],
+                 scan('embvar', '%Q[#@ivar]')
+  end
+
+  def test_float
+    assert_equal [],
+                 scan('float', '')
+    assert_equal ['1.000'],
+                 scan('float', '1.000')
+    assert_equal ['123.456'],
+                 scan('float', '123.456')
+    assert_equal ['1.2345678901234567890123456789'],
+                 scan('float', '1.2345678901234567890123456789')
+    assert_equal ['1.000'],
+                 scan('float', '   1.000# comment')
+    assert_equal ['1.234e5'],
+                 scan('float', '1.234e5')
+    assert_equal ['1.234e1234567890'],
+                 scan('float', '1.234e1234567890')
+    assert_equal ['1.0'],
+                 scan('float', 'm(a,b,1.0,c,d)')
+  end
+
+  def test_rational
+    assert_equal [],
+                 scan('rational', '')
+    assert_equal ['1r', '10r', '10.1r'],
+                 scan('rational', 'm(1r,10r,10.1r)')
+  end
+
+  def test_gvar
+    assert_equal [],
+                 scan('gvar', '')
+    assert_equal ['$a'],
+                 scan('gvar', '$a')
+    assert_equal ['$A'],
+                 scan('gvar', '$A')
+    assert_equal ['$gvar'],
+                 scan('gvar', 'm(lvar, @ivar, @@cvar, $gvar)')
+    assert_equal %w($_ $~ $* $$ $? $! $@ $/ $\\ $; $, $. $= $: $< $> $"),
+                 scan('gvar', 'm($_, $~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $")')
+  end
+
+  def test_ident
+    assert_equal [],
+                 scan('ident', '')
+    assert_equal ['lvar'],
+                 scan('ident', 'lvar')
+    assert_equal ['m', 'lvar'],
+                 scan('ident', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_imaginary
+    assert_equal [],
+                 scan('imaginary', '')
+    assert_equal ['1i', '10ri', '10.0i', '10.1ri'],
+                 scan('imaginary', 'm(1i,10ri,10.0i,10.1ri)')
+  end
+
+  def test_int
+    assert_equal [],
+                 scan('int', '')
+    assert_equal ['1', '10', '100000000000000'],
+                 scan('int', 'm(1,10,100000000000000)')
+  end
+
+  def test_ivar
+    assert_equal [],
+                 scan('ivar', '')
+    assert_equal ['@ivar'],
+                 scan('ivar', '@ivar')
+    assert_equal ['@__ivar__'],
+                 scan('ivar', '@__ivar__')
+    assert_equal ['@IVAR'],
+                 scan('ivar', '@IVAR')
+    assert_equal ['@ivar'],
+                 scan('ivar', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_kw
+    assert_equal [],
+                 scan('kw', '')
+    assert_equal %w(not),
+                 scan('kw', 'not 1')
+    assert_equal %w(and),
+                 scan('kw', '1 and 2')
+    assert_equal %w(or),
+                 scan('kw', '1 or 2')
+    assert_equal %w(if then else end),
+                 scan('kw', 'if 1 then 2 else 3 end')
+    assert_equal %w(if then elsif else end),
+                 scan('kw', 'if 1 then 2 elsif 3 else 4 end')
+    assert_equal %w(unless then end),
+                 scan('kw', 'unless 1 then end')
+    assert_equal %w(if true),
+                 scan('kw', '1 if true')
+    assert_equal %w(unless false),
+                 scan('kw', '2 unless false')
+    assert_equal %w(case when when else end),
+                 scan('kw', 'case n; when 1; when 2; else 3 end')
+    assert_equal %w(while do nil end),
+                 scan('kw', 'while 1 do nil end')
+    assert_equal %w(until do nil end),
+                 scan('kw', 'until 1 do nil end')
+    assert_equal %w(while),
+                 scan('kw', '1 while 2')
+    assert_equal %w(until),
+                 scan('kw', '1 until 2')
+    assert_equal %w(while break next retry end),
+                 scan('kw', 'while 1; break; next; retry end')
+    assert_equal %w(for in next break end),
+                 scan('kw', 'for x in obj; next 1; break 2 end')
+    assert_equal %w(begin rescue retry end),
+                 scan('kw', 'begin 1; rescue; retry; end')
+    assert_equal %w(rescue),
+                 scan('kw', '1 rescue 2')
+    assert_equal %w(def redo return end),
+                 scan('kw', 'def m() redo; return end')
+    assert_equal %w(def yield yield end),
+                 scan('kw', 'def m() yield; yield 1 end')
+    assert_equal %w(def super super super end),
+                 scan('kw', 'def m() super; super(); super(1) end')
+    assert_equal %w(alias),
+                 scan('kw', 'alias a b')
+    assert_equal %w(undef),
+                 scan('kw', 'undef public')
+    assert_equal %w(class end),
+                 scan('kw', 'class A < Object; end')
+    assert_equal %w(module end),
+                 scan('kw', 'module M; end')
+    assert_equal %w(class end),
+                 scan('kw', 'class << obj; end')
+    assert_equal %w(BEGIN),
+                 scan('kw', 'BEGIN { }')
+    assert_equal %w(END),
+                 scan('kw', 'END { }')
+    assert_equal %w(self),
+                 scan('kw', 'self.class')
+    assert_equal %w(nil true false),
+                 scan('kw', 'p(nil, true, false)')
+    assert_equal %w(__FILE__ __LINE__),
+                 scan('kw', 'p __FILE__, __LINE__')
+    assert_equal %w(defined?),
+                 scan('kw', 'defined?(Object)')
+  end
+
+  def test_lbrace
+    assert_equal [],
+                 scan('lbrace', '')
+    assert_equal ['{'],
+                 scan('lbrace', '3.times{ }')
+    assert_equal ['{'],
+                 scan('lbrace', '3.times  { }')
+    assert_equal ['{'],
+                 scan('lbrace', '3.times{}')
+    assert_equal [],
+                 scan('lbrace', '"{}"')
+    assert_equal ['{'],
+                 scan('lbrace', '{1=>2}')
+  end
+
+  def test_rbrace
+    assert_equal [],
+                 scan('rbrace', '')
+    assert_equal ['}'],
+                 scan('rbrace', '3.times{ }')
+    assert_equal ['}'],
+                 scan('rbrace', '3.times  { }')
+    assert_equal ['}'],
+                 scan('rbrace', '3.times{}')
+    assert_equal [],
+                 scan('rbrace', '"{}"')
+    assert_equal ['}'],
+                 scan('rbrace', '{1=>2}')
+  end
+
+  def test_lbracket
+    assert_equal [],
+                 scan('lbracket', '')
+    assert_equal ['['],
+                 scan('lbracket', '[]')
+    assert_equal ['['],
+                 scan('lbracket', 'a[1]')
+    assert_equal [],
+                 scan('lbracket', 'm(%q[])')
+  end
+
+  def test_rbracket
+    assert_equal [],
+                 scan('rbracket', '')
+    assert_equal [']'],
+                 scan('rbracket', '[]')
+    assert_equal [']'],
+                 scan('rbracket', 'a[1]')
+    assert_equal [],
+                 scan('rbracket', 'm(%q[])')
+  end
+
+  def test_lparen
+    assert_equal [],
+                 scan('lparen', '')
+    assert_equal ['('],
+                 scan('lparen', '()')
+    assert_equal ['('],
+                 scan('lparen', 'm()')
+    assert_equal ['('],
+                 scan('lparen', 'm (a)')
+    assert_equal [],
+                 scan('lparen', '"()"')
+    assert_equal [],
+                 scan('lparen', '"%w()"')
+  end
+
+  def test_rparen
+    assert_equal [],
+                 scan('rparen', '')
+    assert_equal [')'],
+                 scan('rparen', '()')
+    assert_equal [')'],
+                 scan('rparen', 'm()')
+    assert_equal [')'],
+                 scan('rparen', 'm (a)')
+    assert_equal [],
+                 scan('rparen', '"()"')
+    assert_equal [],
+                 scan('rparen', '"%w()"')
+  end
+
+  def test_op
+    assert_equal [],
+                 scan('op', '')
+    assert_equal ['|'],
+                 scan('op', '1 | 1')
+    assert_equal ['^'],
+                 scan('op', '1 ^ 1')
+    assert_equal ['&'],
+                 scan('op', '1 & 1')
+    assert_equal ['<=>'],
+                 scan('op', '1 <=> 1')
+    assert_equal ['=='],
+                 scan('op', '1 == 1')
+    assert_equal ['==='],
+                 scan('op', '1 === 1')
+    assert_equal ['=~'],
+                 scan('op', '1 =~ 1')
+    assert_equal ['>'],
+                 scan('op', '1 > 1')
+    assert_equal ['>='],
+                 scan('op', '1 >= 1')
+    assert_equal ['<'],
+                 scan('op', '1 < 1')
+    assert_equal ['<='],
+                 scan('op', '1 <= 1')
+    assert_equal ['<''<'],
+                 scan('op', '1 <''< 1')
+    assert_equal ['>>'],
+                 scan('op', '1 >> 1')
+    assert_equal ['+'],
+                 scan('op', '1 + 1')
+    assert_equal ['-'],
+                 scan('op', '1 - 1')
+    assert_equal ['*'],
+                 scan('op', '1 * 1')
+    assert_equal ['/'],
+                 scan('op', '1 / 1')
+    assert_equal ['%'],
+                 scan('op', '1 % 1')
+    assert_equal ['**'],
+                 scan('op', '1 ** 1')
+    assert_equal ['~'],
+                 scan('op', '~1')
+    assert_equal ['-'],
+                 scan('op', '-a')
+    assert_equal ['+'],
+                 scan('op', '+a')
+    assert_equal ['[]'],
+                 scan('op', ':[]')
+    assert_equal ['[]='],
+                 scan('op', ':[]=')
+    assert_equal [],
+                 scan('op', %q[`make all`])
+  end
+
+  def test_symbeg
+    assert_equal [],
+                 scan('symbeg', '')
+    assert_equal [':'],
+                 scan('symbeg', ':sym')
+    assert_equal [':'],
+                 scan('symbeg', '[1,2,3,:sym]')
+    assert_equal [],
+                 scan('symbeg', '":sym"')
+    assert_equal [],
+                 scan('symbeg', 'a ? b : c')
+  end
+
+  def test_tstring_beg
+    assert_equal [],
+                 scan('tstring_beg', '')
+    assert_equal ['"'],
+                 scan('tstring_beg', '"abcdef"')
+    assert_equal ['%q['],
+                 scan('tstring_beg', '%q[abcdef]')
+    assert_equal ['%Q['],
+                 scan('tstring_beg', '%Q[abcdef]')
+  end
+
+  def test_tstring_content
+    assert_equal [],
+                 scan('tstring_content', '')
+    assert_equal ['abcdef'],
+                 scan('tstring_content', '"abcdef"')
+    assert_equal ['abcdef'],
+                 scan('tstring_content', '%q[abcdef]')
+    assert_equal ['abcdef'],
+                 scan('tstring_content', '%Q[abcdef]')
+    assert_equal ['abc', 'def'],
+                 scan('tstring_content', '"abc#{1}def"')
+    assert_equal ['sym'],
+                 scan('tstring_content', ':"sym"')
+    assert_equal ['a b c'],
+                 scan('tstring_content', ':"a b c"'),
+                 "bug#4544"
+    assert_equal ["a\nb\nc"],
+                 scan('tstring_content', ":'a\nb\nc'"),
+                 "bug#4544"
+  end
+
+  def test_tstring_end
+    assert_equal [],
+                 scan('tstring_end', '')
+    assert_equal ['"'],
+                 scan('tstring_end', '"abcdef"')
+    assert_equal [']'],
+                 scan('tstring_end', '%q[abcdef]')
+    assert_equal [']'],
+                 scan('tstring_end', '%Q[abcdef]')
+  end
+
+  def test_regexp_beg
+    assert_equal [],
+                 scan('regexp_beg', '')
+    assert_equal ['/'],
+                 scan('regexp_beg', '/re/')
+    assert_equal ['%r<'],
+                 scan('regexp_beg', '%r<re>')
+    assert_equal [],
+                 scan('regexp_beg', '5 / 5')
+  end
+
+  def test_regexp_end
+    assert_equal [],
+                 scan('regexp_end', '')
+    assert_equal ['/'],
+                 scan('regexp_end', '/re/')
+    assert_equal ['>'],
+                 scan('regexp_end', '%r<re>')
+  end
+
+  def test_words_beg
+    assert_equal [],
+                 scan('words_beg', '')
+    assert_equal ['%W('],
+                 scan('words_beg', '%W()')
+    assert_equal ['%W('],
+                 scan('words_beg', '%W(w w w)')
+    assert_equal ['%W( '],
+                 scan('words_beg', '%W( w w w )')
+  end
+
+  def test_qwords_beg
+    assert_equal [],
+                 scan('qwords_beg', '')
+    assert_equal ['%w('],
+                 scan('qwords_beg', '%w()')
+    assert_equal ['%w('],
+                 scan('qwords_beg', '%w(w w w)')
+    assert_equal ['%w( '],
+                 scan('qwords_beg', '%w( w w w )')
+  end
+
+  def test_qsymbols_beg
+    assert_equal [],
+                 scan('qsymbols_beg', '')
+    assert_equal ['%i('],
+                 scan('qsymbols_beg', '%i()')
+    assert_equal ['%i('],
+                 scan('qsymbols_beg', '%i(w w w)')
+    assert_equal ['%i( '],
+                 scan('qsymbols_beg', '%i( w w w )')
+  end
+
+  def test_symbols_beg
+    assert_equal [],
+                 scan('symbols_beg', '')
+    assert_equal ['%I('],
+                 scan('symbols_beg', '%I()')
+    assert_equal ['%I('],
+                 scan('symbols_beg', '%I(w w w)')
+    assert_equal ['%I( '],
+                 scan('symbols_beg', '%I( w w w )')
+  end
+
+  # FIXME: Close paren must not present (`words_end' scanner event?).
+  def test_words_sep
+    assert_equal [],
+                 scan('words_sep', '')
+    assert_equal [')'],
+                 scan('words_sep', '%w()')
+    assert_equal [' ', ' ', ')'],
+                 scan('words_sep', '%w(w w w)')
+    assert_equal [' ', ' ', ' )'],
+                 scan('words_sep', '%w( w w w )')
+    assert_equal ["\n", ' ', ' )'],
+                 scan('words_sep', "%w( w\nw w )")
+  end
+
+  def test_heredoc_beg
+    assert_equal [],
+                 scan('heredoc_beg', '')
+    assert_equal ['<<''EOS'],
+                 scan('heredoc_beg', "<<""EOS\nheredoc\nEOS")
+    assert_equal ['<<''EOS'],
+                 scan('heredoc_beg', "<<""EOS\nheredoc\nEOS\n")
+    assert_equal ['<<''EOS'],
+                 scan('heredoc_beg', "<<""EOS\nheredoc\nEOS \n")
+    assert_equal ['<<''-EOS'],
+                 scan('heredoc_beg', "<<""-EOS\nheredoc\n\tEOS \n")
+    assert_equal ['<<''"EOS"'],
+                 scan('heredoc_beg', '<<''"EOS"'"\nheredoc\nEOS")
+    assert_equal ["<<""'EOS'"],
+                 scan('heredoc_beg', "<<""'EOS'\nheredoc\nEOS")
+    assert_equal ['<<''`EOS`'],
+                 scan('heredoc_beg', "<<""`EOS`\nheredoc\nEOS")
+    assert_equal ['<<''" "'],
+                 scan('heredoc_beg', '<<''" "'"\nheredoc\nEOS")
+  end
+
+  def test_tstring_content_HEREDOC
+    assert_equal [],
+                 scan('tstring_content', '')
+    assert_equal ["heredoc\n"],
+                 scan('tstring_content', "<<""EOS\nheredoc\nEOS")
+    assert_equal ["heredoc\n"],
+                 scan('tstring_content', "<<""EOS\nheredoc\nEOS\n")
+    assert_equal ["here\ndoc \nEOS \n"],
+                 scan('tstring_content', "<<""EOS\nhere\ndoc \nEOS \n")
+    assert_equal ["heredoc\n\tEOS \n"],
+                 scan('tstring_content', "<<""-EOS\nheredoc\n\tEOS \n")
+    bug7255 = '[ruby-core:48703]'
+    assert_equal ["there\n""heredoc", "\n"],
+                 scan('tstring_content', "<<""EOS\n""there\n""heredoc\#{foo}\nEOS"),
+                 bug7255
+    assert_equal ["there\n""heredoc", "\n"],
+                 scan('tstring_content', "<<""EOS\n""there\n""heredoc\#@foo\nEOS"),
+                 bug7255
+    bug10392 = '[ruby-dev:48647] [Bug #10392]'
+    assert_equal [" E\n\n"],
+                 scan('tstring_content', "<<""'E'\n E\n\n"),
+                 bug10392
+  end
+
+  def test_heredoc_end
+    assert_equal [],
+                 scan('heredoc_end', '')
+    assert_equal ["EOS"],
+                 scan('heredoc_end', "<<""EOS\nEOS"),
+                 "bug#4543"
+    assert_equal ["EOS"],
+                 scan('heredoc_end', "<<""EOS\nheredoc\nEOS")
+    assert_equal ["EOS\n"],
+                 scan('heredoc_end', "<<""EOS\nheredoc\nEOS\n")
+    assert_equal [],
+                 scan('heredoc_end', "<<""EOS\nheredoc\nEOS \n")
+    assert_equal [],
+                 scan('heredoc_end', "<<""-EOS\nheredoc\n\tEOS \n")
+  end
+
+  def test_semicolon
+    assert_equal [],
+                 scan('semicolon', '')
+    assert_equal %w(;),
+                 scan('semicolon', ';')
+    assert_equal %w(; ;),
+                 scan('semicolon', '; ;')
+    assert_equal %w(; ; ;),
+                 scan('semicolon', 'nil;nil;nil;')
+    assert_equal %w(; ; ;),
+                 scan('semicolon', 'nil;nil;nil;nil')
+    assert_equal [],
+                 scan('semicolon', '";"')
+    assert_equal [],
+                 scan('semicolon', '%w(;)')
+    assert_equal [],
+                 scan('semicolon', '/;/')
+  end
+
+  def test_comment
+    assert_equal [],
+                 scan('comment', '')
+    assert_equal ['# comment'],
+                 scan('comment', '# comment')
+    assert_equal ["# comment\n"],
+                 scan('comment', "# comment\n")
+    assert_equal ["# comment\n"],
+                 scan('comment', "# comment\n1 + 1")
+    assert_equal ["# comment\n"],
+                 scan('comment', "1 + 1 + 1# comment\n1 + 1")
+  end
+
+  def test_embdoc_beg
+    assert_equal [],
+                 scan('embdoc_beg', '')
+    assert_equal ["=begin\n"],
+                 scan('embdoc_beg', "=begin\ndoc\n=end")
+    assert_equal ["=begin \n"],
+                 scan('embdoc_beg', "=begin \ndoc\n=end\n")
+    assert_equal ["=begin comment\n"],
+                 scan('embdoc_beg', "=begin comment\ndoc\n=end\n")
+  end
+
+  def test_embdoc
+    assert_equal [],
+                 scan('embdoc', '')
+    assert_equal ["doc\n"],
+                 scan('embdoc', "=begin\ndoc\n=end")
+    assert_equal ["doc\n"],
+                 scan('embdoc', "=begin\ndoc\n=end\n")
+  end
+
+  def test_embdoc_end
+    assert_equal [],
+                 scan('embdoc_end', '')
+    assert_equal ["=end"],
+                 scan('embdoc_end', "=begin\ndoc\n=end")
+    assert_equal ["=end\n"],
+                 scan('embdoc_end', "=begin\ndoc\n=end\n")
+  end
+
+  def test_sp
+    assert_equal [],
+                 scan('sp', '')
+    assert_equal [' '],
+                 scan('sp', ' ')
+    assert_equal [' '],
+                 scan('sp', ' 1')
+    assert_equal [],
+                 scan('sp', "\n")
+    assert_equal [' '],
+                 scan('sp', " \n")
+    assert_equal [' ', ' '],
+                 scan('sp', "1 + 1")
+    assert_equal [],
+                 scan('sp', "' '")
+    assert_equal [],
+                 scan('sp', "%w(  )")
+    assert_equal [],
+                 scan('sp', "%w(  w  )")
+    assert_equal [],
+                 scan('sp', "p(/ /)")
+  end
+
+  # `nl' event always means End-Of-Statement.
+  def test_nl
+    assert_equal [],
+                 scan('nl', '')
+    assert_equal [],
+                 scan('nl', "\n")
+    assert_equal ["\n"],
+                 scan('nl', "1 + 1\n")
+    assert_equal ["\n", "\n"],
+                 scan('nl', "1 + 1\n2 + 2\n")
+    assert_equal [],
+                 scan('nl', "1 +\n1")
+    assert_equal [],
+                 scan('nl', "1;\n")
+    assert_equal ["\r\n"],
+                 scan('nl', "1 + 1\r\n")
+    assert_equal [],
+                 scan('nl', "1;\r\n")
+  end
+
+  def test_ignored_nl
+    assert_equal [],
+                 scan('ignored_nl', '')
+    assert_equal ["\n"],
+                 scan('ignored_nl', "\n")
+    assert_equal [],
+                 scan('ignored_nl', "1 + 1\n")
+    assert_equal [],
+                 scan('ignored_nl', "1 + 1\n2 + 2\n")
+    assert_equal ["\n"],
+                 scan('ignored_nl', "1 +\n1")
+    assert_equal ["\n"],
+                 scan('ignored_nl', "1;\n")
+    assert_equal [],
+                 scan('ignored_nl', "1 + 1\r\n")
+    assert_equal ["\r\n"],
+                 scan('ignored_nl', "1;\r\n")
+  end
+
+  def test___end__
+    assert_equal [],
+                 scan('__end__', "")
+    assert_equal ["__END__"],
+                 scan('__end__', "__END__")
+    assert_equal ["__END__\n"],
+                 scan('__end__', "__END__\n")
+    assert_equal ["__END__\n"],
+                 Ripper.tokenize("__END__\njunk junk junk")
+    assert_equal ["__END__"],
+                 scan('__end__', "1\n__END__")
+    assert_equal [],
+                 scan('__end__', "print('__END__')")
+  end
+
+  def test_CHAR
+    assert_equal [],
+                 scan('CHAR', "")
+    assert_equal ["?a"],
+                 scan('CHAR', "?a")
+    assert_equal [],
+                 scan('CHAR', "@ivar")
+  end
+
+  def test_label
+    assert_equal %w(foo:),
+                 scan('label', '{foo: 1}')
+  end
+
+  def test_label_end
+    assert_equal %w(":),
+                 scan('label_end', '{"foo-bar": 1}')
+  end
+
+  def test_tlambda
+    assert_equal %w(->),
+                 scan('tlambda', '->{}')
+  end
+
+  def test_tlambeg
+    assert_equal %w({),
+                 scan('tlambeg', '-> {}')
+  end
+
+  def test_tlambda_arg
+    assert_equal %w(),
+                 scan('tlambda_arg', '-> {}')
+  end
+
+end if ripper_test
diff --git a/jni/ruby/test/ripper/test_sexp.rb b/jni/ruby/test/ripper/test_sexp.rb
new file mode 100644
index 0000000..2c5bcda
--- /dev/null
+++ b/jni/ruby/test/ripper/test_sexp.rb
@@ -0,0 +1,44 @@
+begin
+  require 'ripper'
+  require 'test/unit'
+  ripper_test = true
+  module TestRipper; end
+rescue LoadError
+end
+
+class TestRipper::Sexp < Test::Unit::TestCase
+  def test_compile_error
+    assert_nil Ripper.sexp("/")
+    assert_nil Ripper.sexp("-")
+    assert_nil Ripper.sexp("+")
+    assert_nil Ripper.sexp("*")
+    assert_nil Ripper.sexp("end")
+    assert_nil Ripper.sexp("end 1")
+    assert_nil Ripper.sexp("/*")
+    assert_nil Ripper.sexp("/*/")
+    assert_nil Ripper.sexp("/+/")
+  end
+
+  def test_regexp_content
+    sexp = Ripper.sexp('//')
+    assert_nil search_sexp(:@tstring_content, search_sexp(:regexp_literal, sexp))
+
+    sexp = Ripper.sexp('/foo/')
+    assert_equal 'foo', search_sexp(:@tstring_content, search_sexp(:regexp_literal, sexp))[1]
+
+    sexp = Ripper.sexp("/foo\nbar/")
+    assert_equal "foo\nbar", search_sexp(:@tstring_content, search_sexp(:regexp_literal, sexp))[1]
+
+    sexp = Ripper.sexp('/(?<n>a(b|\g<n>))/')
+    assert_equal '(?<n>a(b|\g<n>))', search_sexp(:@tstring_content, search_sexp(:regexp_literal, sexp))[1]
+  end
+
+  def search_sexp(sym, sexp)
+    return sexp if !sexp or sexp[0] == sym
+    sexp.find do |e|
+      if Array === e and e = search_sexp(sym, e)
+        return e
+      end
+    end
+  end
+end if ripper_test