Fresh start

8 files changed, 3871 insertions, 0 deletions

diff --git a/jni/ruby/lib/rdoc/parser/c.rb b/jni/ruby/lib/rdoc/parser/c.rb
new file mode 100644
index 0000000..fd336f5
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/c.rb
@@ -0,0 +1,1229 @@
+require 'tsort'
+
+##
+# RDoc::Parser::C attempts to parse C extension files.  It looks for
+# the standard patterns that you find in extensions: <tt>rb_define_class,
+# rb_define_method</tt> and so on.  It tries to find the corresponding
+# C source for the methods and extract comments, but if we fail
+# we don't worry too much.
+#
+# The comments associated with a Ruby method are extracted from the C
+# comment block associated with the routine that _implements_ that
+# method, that is to say the method whose name is given in the
+# <tt>rb_define_method</tt> call. For example, you might write:
+#
+#   /*
+#    * Returns a new array that is a one-dimensional flattening of this
+#    * array (recursively). That is, for every element that is an array,
+#    * extract its elements into the new array.
+#    *
+#    *    s = [ 1, 2, 3 ]           #=> [1, 2, 3]
+#    *    t = [ 4, 5, 6, [7, 8] ]   #=> [4, 5, 6, [7, 8]]
+#    *    a = [ s, t, 9, 10 ]       #=> [[1, 2, 3], [4, 5, 6, [7, 8]], 9, 10]
+#    *    a.flatten                 #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+#    */
+#    static VALUE
+#    rb_ary_flatten(ary)
+#        VALUE ary;
+#    {
+#        ary = rb_obj_dup(ary);
+#        rb_ary_flatten_bang(ary);
+#        return ary;
+#    }
+#
+#    ...
+#
+#    void
+#    Init_Array()
+#    {
+#      ...
+#      rb_define_method(rb_cArray, "flatten", rb_ary_flatten, 0);
+#
+# Here RDoc will determine from the rb_define_method line that there's a
+# method called "flatten" in class Array, and will look for the implementation
+# in the method rb_ary_flatten. It will then use the comment from that
+# method in the HTML output. This method must be in the same source file
+# as the rb_define_method.
+#
+# The comment blocks may include special directives:
+#
+# [Document-class: +name+]
+#   Documentation for the named class.
+#
+# [Document-module: +name+]
+#   Documentation for the named module.
+#
+# [Document-const: +name+]
+#   Documentation for the named +rb_define_const+.
+#
+#   Constant values can be supplied on the first line of the comment like so:
+#
+#     /* 300: The highest possible score in bowling */
+#     rb_define_const(cFoo, "PERFECT", INT2FIX(300));
+#
+#   The value can contain internal colons so long as they are escaped with a \
+#
+# [Document-global: +name+]
+#   Documentation for the named +rb_define_global_const+
+#
+# [Document-variable: +name+]
+#   Documentation for the named +rb_define_variable+
+#
+# [Document-method: +method_name+]
+#   Documentation for the named method.  Use this when the method name is
+#   unambiguous.
+#
+# [Document-method: <tt>ClassName::method_name<tt>]
+#   Documentation for a singleton method in the given class.  Use this when
+#   the method name alone is ambiguous.
+#
+# [Document-method: <tt>ClassName#method_name<tt>]
+#   Documentation for a instance method in the given class.  Use this when the
+#   method name alone is ambiguous.
+#
+# [Document-attr: +name+]
+#   Documentation for the named attribute.
+#
+# [call-seq:  <i>text up to an empty line</i>]
+#   Because C source doesn't give descriptive names to Ruby-level parameters,
+#   you need to document the calling sequence explicitly
+#
+# In addition, RDoc assumes by default that the C method implementing a
+# Ruby function is in the same source file as the rb_define_method call.
+# If this isn't the case, add the comment:
+#
+#   rb_define_method(....);  // in filename
+#
+# As an example, we might have an extension that defines multiple classes
+# in its Init_xxx method. We could document them using
+#
+#   /*
+#    * Document-class:  MyClass
+#    *
+#    * Encapsulate the writing and reading of the configuration
+#    * file. ...
+#    */
+#
+#   /*
+#    * Document-method: read_value
+#    *
+#    * call-seq:
+#    *   cfg.read_value(key)            -> value
+#    *   cfg.read_value(key} { |key| }  -> value
+#    *
+#    * Return the value corresponding to +key+ from the configuration.
+#    * In the second form, if the key isn't found, invoke the
+#    * block and return its value.
+#    */
+
+class RDoc::Parser::C < RDoc::Parser
+
+  parse_files_matching(/\.(?:([CcHh])\1?|c([+xp])\2|y)\z/)
+
+  include RDoc::Text
+
+  ##
+  # Maps C variable names to names of Ruby classes or modules
+
+  attr_reader :classes
+
+  ##
+  # C file the parser is parsing
+
+  attr_accessor :content
+
+  ##
+  # Dependencies from a missing enclosing class to the classes in
+  # missing_dependencies that depend upon it.
+
+  attr_reader :enclosure_dependencies
+
+  ##
+  # Maps C variable names to names of Ruby classes (and singleton classes)
+
+  attr_reader :known_classes
+
+  ##
+  # Classes found while parsing the C file that were not yet registered due to
+  # a missing enclosing class.  These are processed by do_missing
+
+  attr_reader :missing_dependencies
+
+  ##
+  # Maps C variable names to names of Ruby singleton classes
+
+  attr_reader :singleton_classes
+
+  ##
+  # The TopLevel items in the parsed file belong to
+
+  attr_reader :top_level
+
+  ##
+  # Prepares for parsing a C file.  See RDoc::Parser#initialize for details on
+  # the arguments.
+
+  def initialize top_level, file_name, content, options, stats
+    super
+
+    @known_classes = RDoc::KNOWN_CLASSES.dup
+    @content = handle_tab_width handle_ifdefs_in @content
+    @file_dir = File.dirname @file_name
+
+    @classes           = load_variable_map :c_class_variables
+    @singleton_classes = load_variable_map :c_singleton_class_variables
+
+    # class_variable => { function => [method, ...] }
+    @methods = Hash.new { |h, f| h[f] = Hash.new { |i, m| i[m] = [] } }
+
+    # missing variable => [handle_class_module arguments]
+    @missing_dependencies = {}
+
+    # missing enclosure variable => [dependent handle_class_module arguments]
+    @enclosure_dependencies = Hash.new { |h, k| h[k] = [] }
+    @enclosure_dependencies.instance_variable_set :@missing_dependencies,
+                                                  @missing_dependencies
+
+    @enclosure_dependencies.extend TSort
+
+    def @enclosure_dependencies.tsort_each_node &block
+      each_key(&block)
+    rescue TSort::Cyclic => e
+      cycle_vars = e.message.scan(/"(.*?)"/).flatten
+
+      cycle = cycle_vars.sort.map do |var_name|
+        delete var_name
+
+        var_name, type, mod_name, = @missing_dependencies[var_name]
+
+        "#{type} #{mod_name} (#{var_name})"
+      end.join ', '
+
+      warn "Unable to create #{cycle} due to a cyclic class or module creation"
+
+      retry
+    end
+
+    def @enclosure_dependencies.tsort_each_child node, &block
+      fetch(node, []).each(&block)
+    end
+  end
+
+  ##
+  # Removes duplicate call-seq entries for methods using the same
+  # implementation.
+
+  def deduplicate_call_seq
+    @methods.each do |var_name, functions|
+      class_name = @known_classes[var_name]
+      class_obj  = find_class var_name, class_name
+
+      functions.each_value do |method_names|
+        next if method_names.length == 1
+
+        method_names.each do |method_name|
+          deduplicate_method_name class_obj, method_name
+        end
+      end
+    end
+  end
+
+  ##
+  # If two ruby methods share a C implementation (and comment) this
+  # deduplicates the examples in the call_seq for the method to reduce
+  # confusion in the output.
+
+  def deduplicate_method_name class_obj, method_name # :nodoc:
+    return unless
+      method = class_obj.method_list.find { |m| m.name == method_name }
+    return unless call_seq = method.call_seq
+
+    method_name = method_name[0, 1] if method_name =~ /\A\[/
+
+    entries = call_seq.split "\n"
+
+    matching = entries.select do |entry|
+      entry =~ /^\w*\.?#{Regexp.escape method_name}/ or
+        entry =~ /\s#{Regexp.escape method_name}\s/
+    end
+
+    method.call_seq = matching.join "\n"
+  end
+
+  ##
+  # Scans #content for rb_define_alias
+
+  def do_aliases
+    @content.scan(/rb_define_alias\s*\(
+                   \s*(\w+),
+                   \s*"(.+?)",
+                   \s*"(.+?)"
+                   \s*\)/xm) do |var_name, new_name, old_name|
+      class_name = @known_classes[var_name]
+
+      unless class_name then
+        @options.warn "Enclosing class or module %p for alias %s %s is not known" % [
+          var_name, new_name, old_name]
+        next
+      end
+
+      class_obj = find_class var_name, class_name
+
+      al = RDoc::Alias.new '', old_name, new_name, ''
+      al.singleton = @singleton_classes.key? var_name
+
+      comment = find_alias_comment var_name, new_name, old_name
+
+      comment.normalize
+
+      al.comment = comment
+
+      al.record_location @top_level
+
+      class_obj.add_alias al
+      @stats.add_alias al
+    end
+  end
+
+  ##
+  # Scans #content for rb_attr and rb_define_attr
+
+  def do_attrs
+    @content.scan(/rb_attr\s*\(
+                   \s*(\w+),
+                   \s*([\w"()]+),
+                   \s*([01]),
+                   \s*([01]),
+                   \s*\w+\);/xm) do |var_name, attr_name, read, write|
+      handle_attr var_name, attr_name, read, write
+    end
+
+    @content.scan(%r%rb_define_attr\(
+                             \s*([\w\.]+),
+                             \s*"([^"]+)",
+                             \s*(\d+),
+                             \s*(\d+)\s*\);
+                %xm) do |var_name, attr_name, read, write|
+      handle_attr var_name, attr_name, read, write
+    end
+  end
+
+  ##
+  # Scans #content for boot_defclass
+
+  def do_boot_defclass
+    @content.scan(/(\w+)\s*=\s*boot_defclass\s*\(\s*"(\w+?)",\s*(\w+?)\s*\)/) do
+      |var_name, class_name, parent|
+      parent = nil if parent == "0"
+      handle_class_module(var_name, :class, class_name, parent, nil)
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_class, boot_defclass, rb_define_class_under
+  # and rb_singleton_class
+
+  def do_classes
+    do_boot_defclass
+    do_define_class
+    do_define_class_under
+    do_singleton_class
+    do_struct_define_without_accessor
+  end
+
+  ##
+  # Scans #content for rb_define_variable, rb_define_readonly_variable,
+  # rb_define_const and rb_define_global_const
+
+  def do_constants
+    @content.scan(%r%\Wrb_define_
+                   ( variable          |
+                     readonly_variable |
+                     const             |
+                     global_const        )
+               \s*\(
+                 (?:\s*(\w+),)?
+                 \s*"(\w+)",
+                 \s*(.*?)\s*\)\s*;
+                 %xm) do |type, var_name, const_name, definition|
+      var_name = "rb_cObject" if !var_name or var_name == "rb_mKernel"
+      handle_constants type, var_name, const_name, definition
+    end
+
+    @content.scan(%r%
+                  \Wrb_curses_define_const
+                  \s*\(
+                    \s*
+                    (\w+)
+                    \s*
+                  \)
+                  \s*;%xm) do |consts|
+      const = consts.first
+
+      handle_constants 'const', 'mCurses', const, "UINT2NUM(#{const})"
+    end
+
+    @content.scan(%r%
+                  \Wrb_file_const
+                  \s*\(
+                    \s*
+                    "([^"]+)",
+                    \s*
+                    (.*?)
+                    \s*
+                  \)
+                  \s*;%xm) do |name, value|
+      handle_constants 'const', 'rb_mFConst', name, value
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_class
+
+  def do_define_class
+    # The '.' lets us handle SWIG-generated files
+    @content.scan(/([\w\.]+)\s* = \s*rb_define_class\s*
+              \(
+                 \s*"(\w+)",
+                 \s*(\w+)\s*
+              \)/mx) do |var_name, class_name, parent|
+      handle_class_module(var_name, :class, class_name, parent, nil)
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_class_under
+
+  def do_define_class_under
+    @content.scan(/([\w\.]+)\s* =                  # var_name
+                   \s*rb_define_class_under\s*
+                   \(
+                     \s* (\w+),                    # under
+                     \s* "(\w+)",                  # class_name
+                     \s*
+                     (?:
+                       ([\w\*\s\(\)\.\->]+) |      # parent_name
+                       rb_path2class\("([\w:]+)"\) # path
+                     )
+                     \s*
+                   \)
+                  /mx) do |var_name, under, class_name, parent_name, path|
+      parent = path || parent_name
+
+      handle_class_module var_name, :class, class_name, parent, under
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_module
+
+  def do_define_module
+    @content.scan(/(\w+)\s* = \s*rb_define_module\s*\(\s*"(\w+)"\s*\)/mx) do
+      |var_name, class_name|
+      handle_class_module(var_name, :module, class_name, nil, nil)
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_module_under
+
+  def do_define_module_under
+    @content.scan(/(\w+)\s* = \s*rb_define_module_under\s*
+              \(
+                 \s*(\w+),
+                 \s*"(\w+)"
+              \s*\)/mx) do |var_name, in_module, class_name|
+      handle_class_module(var_name, :module, class_name, nil, in_module)
+    end
+  end
+
+  ##
+  # Scans #content for rb_include_module
+
+  def do_includes
+    @content.scan(/rb_include_module\s*\(\s*(\w+?),\s*(\w+?)\s*\)/) do |c,m|
+      next unless cls = @classes[c]
+      m = @known_classes[m] || m
+
+      comment = RDoc::Comment.new '', @top_level
+      incl = cls.add_include RDoc::Include.new(m, comment)
+      incl.record_location @top_level
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_method, rb_define_singleton_method,
+  # rb_define_module_function, rb_define_private_method,
+  # rb_define_global_function and define_filetest_function
+
+  def do_methods
+    @content.scan(%r%rb_define_
+                   (
+                      singleton_method |
+                      method           |
+                      module_function  |
+                      private_method
+                   )
+                   \s*\(\s*([\w\.]+),
+                     \s*"([^"]+)",
+                     \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\(|\(METHOD\))?(\w+)\)?,
+                     \s*(-?\w+)\s*\)
+                   (?:;\s*/[*/]\s+in\s+(\w+?\.(?:cpp|c|y)))?
+                 %xm) do |type, var_name, meth_name, function, param_count, source_file|
+
+      # Ignore top-object and weird struct.c dynamic stuff
+      next if var_name == "ruby_top_self"
+      next if var_name == "nstr"
+
+      var_name = "rb_cObject" if var_name == "rb_mKernel"
+      handle_method(type, var_name, meth_name, function, param_count,
+                    source_file)
+    end
+
+    @content.scan(%r%rb_define_global_function\s*\(
+                             \s*"([^"]+)",
+                             \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
+                             \s*(-?\w+)\s*\)
+                (?:;\s*/[*/]\s+in\s+(\w+?\.[cy]))?
+                %xm) do |meth_name, function, param_count, source_file|
+      handle_method("method", "rb_mKernel", meth_name, function, param_count,
+                    source_file)
+    end
+
+    @content.scan(/define_filetest_function\s*\(
+                     \s*"([^"]+)",
+                     \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
+                     \s*(-?\w+)\s*\)/xm) do |meth_name, function, param_count|
+
+      handle_method("method", "rb_mFileTest", meth_name, function, param_count)
+      handle_method("singleton_method", "rb_cFile", meth_name, function,
+                    param_count)
+    end
+  end
+
+  ##
+  # Creates classes and module that were missing were defined due to the file
+  # order being different than the declaration order.
+
+  def do_missing
+    return if @missing_dependencies.empty?
+
+    @enclosure_dependencies.tsort.each do |in_module|
+      arguments = @missing_dependencies.delete in_module
+
+      next unless arguments # dependency on existing class
+
+      handle_class_module(*arguments)
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_module and rb_define_module_under
+
+  def do_modules
+    do_define_module
+    do_define_module_under
+  end
+
+  ##
+  # Scans #content for rb_singleton_class
+
+  def do_singleton_class
+    @content.scan(/([\w\.]+)\s* = \s*rb_singleton_class\s*
+                  \(
+                    \s*(\w+)
+                  \s*\)/mx) do |sclass_var, class_var|
+      handle_singleton sclass_var, class_var
+    end
+  end
+
+  ##
+  # Scans #content for struct_define_without_accessor
+
+  def do_struct_define_without_accessor
+    @content.scan(/([\w\.]+)\s* = \s*rb_struct_define_without_accessor\s*
+              \(
+                 \s*"(\w+)",  # Class name
+                 \s*(\w+),    # Parent class
+                 \s*\w+,      # Allocation function
+                 (\s*"\w+",)* # Attributes
+                 \s*NULL
+              \)/mx) do |var_name, class_name, parent|
+      handle_class_module(var_name, :class, class_name, parent, nil)
+    end
+  end
+
+  ##
+  # Finds the comment for an alias on +class_name+ from +new_name+ to
+  # +old_name+
+
+  def find_alias_comment class_name, new_name, old_name
+    content =~ %r%((?>/\*.*?\*/\s+))
+                  rb_define_alias\(\s*#{Regexp.escape class_name}\s*,
+                                   \s*"#{Regexp.escape new_name}"\s*,
+                                   \s*"#{Regexp.escape old_name}"\s*\);%xm
+
+    RDoc::Comment.new($1 || '', @top_level)
+  end
+
+  ##
+  # Finds a comment for rb_define_attr, rb_attr or Document-attr.
+  #
+  # +var_name+ is the C class variable the attribute is defined on.
+  # +attr_name+ is the attribute's name.
+  #
+  # +read+ and +write+ are the read/write flags ('1' or '0').  Either both or
+  # neither must be provided.
+
+  def find_attr_comment var_name, attr_name, read = nil, write = nil
+    attr_name = Regexp.escape attr_name
+
+    rw = if read and write then
+           /\s*#{read}\s*,\s*#{write}\s*/xm
+         else
+           /.*?/m
+         end
+
+    comment = if @content =~ %r%((?>/\*.*?\*/\s+))
+                                rb_define_attr\((?:\s*#{var_name},)?\s*
+                                                "#{attr_name}"\s*,
+                                                #{rw}\)\s*;%xm then
+                $1
+              elsif @content =~ %r%((?>/\*.*?\*/\s+))
+                                   rb_attr\(\s*#{var_name}\s*,
+                                            \s*#{attr_name}\s*,
+                                            #{rw},.*?\)\s*;%xm then
+                $1
+              elsif @content =~ %r%(/\*.*?(?:\s*\*\s*)?)
+                                   Document-attr:\s#{attr_name}\s*?\n
+                                   ((?>(.|\n)*?\*/))%x then
+                "#{$1}\n#{$2}"
+              else
+                ''
+              end
+
+    RDoc::Comment.new comment, @top_level
+  end
+
+  ##
+  # Find the C code corresponding to a Ruby method
+
+  def find_body class_name, meth_name, meth_obj, file_content, quiet = false
+    case file_content
+    when %r%((?>/\*.*?\*/\s*)?)
+            ((?:(?:\w+)\s+)?
+             (?:intern\s+)?VALUE\s+#{meth_name}
+             \s*(\([^)]*\))([^;]|$))%xm then
+      comment = RDoc::Comment.new $1, @top_level
+      body = $2
+      offset, = $~.offset(2)
+
+      comment.remove_private if comment
+
+      # try to find the whole body
+      body = $& if /#{Regexp.escape body}[^(]*?\{.*?^\}/m =~ file_content
+
+      # The comment block may have been overridden with a 'Document-method'
+      # block. This happens in the interpreter when multiple methods are
+      # vectored through to the same C method but those methods are logically
+      # distinct (for example Kernel.hash and Kernel.object_id share the same
+      # implementation
+
+      override_comment = find_override_comment class_name, meth_obj
+      comment = override_comment if override_comment
+
+      comment.normalize
+      find_modifiers comment, meth_obj if comment
+
+      #meth_obj.params = params
+      meth_obj.start_collecting_tokens
+      tk = RDoc::RubyToken::Token.new nil, 1, 1
+      tk.set_text body
+      meth_obj.add_token tk
+      meth_obj.comment = comment
+      meth_obj.offset  = offset
+      meth_obj.line    = file_content[0, offset].count("\n") + 1
+
+      body
+    when %r%((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+#{meth_name}\s+(\w+))%m then
+      comment = RDoc::Comment.new $1, @top_level
+      body = $2
+      offset = $~.offset(2).first
+
+      find_body class_name, $3, meth_obj, file_content, true
+
+      comment.normalize
+      find_modifiers comment, meth_obj
+
+      meth_obj.start_collecting_tokens
+      tk = RDoc::RubyToken::Token.new nil, 1, 1
+      tk.set_text body
+      meth_obj.add_token tk
+      meth_obj.comment = comment
+      meth_obj.offset  = offset
+      meth_obj.line    = file_content[0, offset].count("\n") + 1
+
+      body
+    when %r%^\s*\#\s*define\s+#{meth_name}\s+(\w+)%m then
+      # with no comment we hope the aliased definition has it and use it's
+      # definition
+
+      body = find_body(class_name, $1, meth_obj, file_content, true)
+
+      return body if body
+
+      @options.warn "No definition for #{meth_name}"
+      false
+    else # No body, but might still have an override comment
+      comment = find_override_comment class_name, meth_obj
+
+      if comment then
+        comment.normalize
+        find_modifiers comment, meth_obj
+        meth_obj.comment = comment
+
+        ''
+      else
+        @options.warn "No definition for #{meth_name}"
+        false
+      end
+    end
+  end
+
+  ##
+  # Finds a RDoc::NormalClass or RDoc::NormalModule for +raw_name+
+
+  def find_class(raw_name, name)
+    unless @classes[raw_name]
+      if raw_name =~ /^rb_m/
+        container = @top_level.add_module RDoc::NormalModule, name
+      else
+        container = @top_level.add_class RDoc::NormalClass, name
+      end
+
+      container.record_location @top_level
+      @classes[raw_name] = container
+    end
+    @classes[raw_name]
+  end
+
+  ##
+  # Look for class or module documentation above Init_+class_name+(void),
+  # in a Document-class +class_name+ (or module) comment or above an
+  # rb_define_class (or module).  If a comment is supplied above a matching
+  # Init_ and a rb_define_class the Init_ comment is used.
+  #
+  #   /*
+  #    * This is a comment for Foo
+  #    */
+  #   Init_Foo(void) {
+  #       VALUE cFoo = rb_define_class("Foo", rb_cObject);
+  #   }
+  #
+  #   /*
+  #    * Document-class: Foo
+  #    * This is a comment for Foo
+  #    */
+  #   Init_foo(void) {
+  #       VALUE cFoo = rb_define_class("Foo", rb_cObject);
+  #   }
+  #
+  #   /*
+  #    * This is a comment for Foo
+  #    */
+  #   VALUE cFoo = rb_define_class("Foo", rb_cObject);
+
+  def find_class_comment class_name, class_mod
+    comment = nil
+
+    if @content =~ %r%
+        ((?>/\*.*?\*/\s+))
+        (static\s+)?
+        void\s+
+        Init_#{class_name}\s*(?:_\(\s*)?\(\s*(?:void\s*)?\)%xmi then
+      comment = $1.sub(%r%Document-(?:class|module):\s+#{class_name}%, '')
+    elsif @content =~ %r%Document-(?:class|module):\s+#{class_name}\s*?
+                         (?:<\s+[:,\w]+)?\n((?>.*?\*/))%xm then
+      comment = "/*\n#{$1}"
+    elsif @content =~ %r%((?>/\*.*?\*/\s+))
+                         ([\w\.\s]+\s* = \s+)?rb_define_(class|module)[\t (]*?"(#{class_name})"%xm then
+      comment = $1
+    elsif @content =~ %r%((?>/\*.*?\*/\s+))
+                         ([\w\. \t]+ = \s+)?rb_define_(class|module)_under[\t\w, (]*?"(#{class_name.split('::').last})"%xm then
+      comment = $1
+    else
+      comment = ''
+    end
+
+    comment = RDoc::Comment.new comment, @top_level
+    comment.normalize
+
+    look_for_directives_in class_mod, comment
+
+    class_mod.add_comment comment, @top_level
+  end
+
+  ##
+  # Finds a comment matching +type+ and +const_name+ either above the
+  # comment or in the matching Document- section.
+
+  def find_const_comment(type, const_name, class_name = nil)
+    comment = if @content =~ %r%((?>^\s*/\*.*?\*/\s+))
+                             rb_define_#{type}\((?:\s*(\w+),)?\s*
+                                                "#{const_name}"\s*,
+                                                .*?\)\s*;%xmi then
+                $1
+              elsif class_name and
+                    @content =~ %r%Document-(?:const|global|variable):\s
+                                   #{class_name}::#{const_name}
+                                   \s*?\n((?>.*?\*/))%xm then
+                "/*\n#{$1}"
+              elsif @content =~ %r%Document-(?:const|global|variable):
+                                   \s#{const_name}
+                                   \s*?\n((?>.*?\*/))%xm then
+                "/*\n#{$1}"
+              else
+                ''
+              end
+
+    RDoc::Comment.new comment, @top_level
+  end
+
+  ##
+  # Handles modifiers in +comment+ and updates +meth_obj+ as appropriate.
+
+  def find_modifiers comment, meth_obj
+    comment.normalize
+    comment.extract_call_seq meth_obj
+
+    look_for_directives_in meth_obj, comment
+  end
+
+  ##
+  # Finds a <tt>Document-method</tt> override for +meth_obj+ on +class_name+
+
+  def find_override_comment class_name, meth_obj
+    name = Regexp.escape meth_obj.name
+    prefix = Regexp.escape meth_obj.name_prefix
+
+    comment = if @content =~ %r%Document-method:
+                                \s+#{class_name}#{prefix}#{name}
+                                \s*?\n((?>.*?\*/))%xm then
+                "/*#{$1}"
+              elsif @content =~ %r%Document-method:
+                                   \s#{name}\s*?\n((?>.*?\*/))%xm then
+                "/*#{$1}"
+              end
+
+    return unless comment
+
+    RDoc::Comment.new comment, @top_level
+  end
+
+  ##
+  # Creates a new RDoc::Attr +attr_name+ on class +var_name+ that is either
+  # +read+, +write+ or both
+
+  def handle_attr(var_name, attr_name, read, write)
+    rw = ''
+    rw << 'R' if '1' == read
+    rw << 'W' if '1' == write
+
+    class_name = @known_classes[var_name]
+
+    return unless class_name
+
+    class_obj = find_class var_name, class_name
+
+    return unless class_obj
+
+    comment = find_attr_comment var_name, attr_name
+    comment.normalize
+
+    name = attr_name.gsub(/rb_intern\("([^"]+)"\)/, '\1')
+
+    attr = RDoc::Attr.new '', name, rw, comment
+
+    attr.record_location @top_level
+    class_obj.add_attribute attr
+    @stats.add_attribute attr
+  end
+
+  ##
+  # Creates a new RDoc::NormalClass or RDoc::NormalModule based on +type+
+  # named +class_name+ in +parent+ which was assigned to the C +var_name+.
+
+  def handle_class_module(var_name, type, class_name, parent, in_module)
+    parent_name = @known_classes[parent] || parent
+
+    if in_module then
+      enclosure = @classes[in_module] || @store.find_c_enclosure(in_module)
+
+      if enclosure.nil? and enclosure = @known_classes[in_module] then
+        enc_type = /^rb_m/ =~ in_module ? :module : :class
+        handle_class_module in_module, enc_type, enclosure, nil, nil
+        enclosure = @classes[in_module]
+      end
+
+      unless enclosure then
+        @enclosure_dependencies[in_module] << var_name
+        @missing_dependencies[var_name] =
+          [var_name, type, class_name, parent, in_module]
+
+        return
+      end
+    else
+      enclosure = @top_level
+    end
+
+    if type == :class then
+      full_name = if RDoc::ClassModule === enclosure then
+                    enclosure.full_name + "::#{class_name}"
+                  else
+                    class_name
+                  end
+
+      if @content =~ %r%Document-class:\s+#{full_name}\s*<\s+([:,\w]+)% then
+        parent_name = $1
+      end
+
+      cm = enclosure.add_class RDoc::NormalClass, class_name, parent_name
+    else
+      cm = enclosure.add_module RDoc::NormalModule, class_name
+    end
+
+    cm.record_location enclosure.top_level
+
+    find_class_comment cm.full_name, cm
+
+    case cm
+    when RDoc::NormalClass
+      @stats.add_class cm
+    when RDoc::NormalModule
+      @stats.add_module cm
+    end
+
+    @classes[var_name] = cm
+    @known_classes[var_name] = cm.full_name
+    @store.add_c_enclosure var_name, cm
+  end
+
+  ##
+  # Adds constants.  By providing some_value: at the start of the comment you
+  # can override the C value of the comment to give a friendly definition.
+  #
+  #   /* 300: The perfect score in bowling */
+  #   rb_define_const(cFoo, "PERFECT", INT2FIX(300);
+  #
+  # Will override <tt>INT2FIX(300)</tt> with the value +300+ in the output
+  # RDoc.  Values may include quotes and escaped colons (\:).
+
+  def handle_constants(type, var_name, const_name, definition)
+    class_name = @known_classes[var_name]
+
+    return unless class_name
+
+    class_obj = find_class var_name, class_name
+
+    unless class_obj then
+      @options.warn 'Enclosing class or module %p is not known' % [const_name]
+      return
+    end
+
+    comment = find_const_comment type, const_name, class_name
+    comment.normalize
+
+    # In the case of rb_define_const, the definition and comment are in
+    # "/* definition: comment */" form.  The literal ':' and '\' characters
+    # can be escaped with a backslash.
+    if type.downcase == 'const' then
+      no_match, new_definition, new_comment = comment.text.split(/(\A.*):/)
+
+      if no_match and no_match.empty? then
+        if new_definition.empty? then # Default to literal C definition
+          new_definition = definition
+        else
+          new_definition.gsub!("\:", ":")
+          new_definition.gsub!("\\", '\\')
+        end
+
+        new_definition.sub!(/\A(\s+)/, '')
+
+        new_comment = "#{$1}#{new_comment.lstrip}"
+
+        new_comment = RDoc::Comment.new new_comment, @top_level
+
+        con = RDoc::Constant.new const_name, new_definition, new_comment
+      else
+        con = RDoc::Constant.new const_name, definition, comment
+      end
+    else
+      con = RDoc::Constant.new const_name, definition, comment
+    end
+
+    con.record_location @top_level
+    @stats.add_constant con
+    class_obj.add_constant con
+  end
+
+  ##
+  # Removes #ifdefs that would otherwise confuse us
+
+  def handle_ifdefs_in(body)
+    body.gsub(/^#ifdef HAVE_PROTOTYPES.*?#else.*?\n(.*?)#endif.*?\n/m, '\1')
+  end
+
+  ##
+  # Adds an RDoc::AnyMethod +meth_name+ defined on a class or module assigned
+  # to +var_name+.  +type+ is the type of method definition function used.
+  # +singleton_method+ and +module_function+ create a singleton method.
+
+  def handle_method(type, var_name, meth_name, function, param_count,
+                    source_file = nil)
+    class_name = @known_classes[var_name]
+    singleton  = @singleton_classes.key? var_name
+
+    @methods[var_name][function] << meth_name
+
+    return unless class_name
+
+    class_obj = find_class var_name, class_name
+
+    if class_obj then
+      if meth_name == 'initialize' then
+        meth_name = 'new'
+        singleton = true
+        type = 'method' # force public
+      end
+
+      meth_obj = RDoc::AnyMethod.new '', meth_name
+      meth_obj.c_function = function
+      meth_obj.singleton =
+        singleton || %w[singleton_method module_function].include?(type)
+
+      p_count = Integer(param_count) rescue -1
+
+      if source_file then
+        file_name = File.join @file_dir, source_file
+
+        if File.exist? file_name then
+          file_content = File.read file_name
+        else
+          @options.warn "unknown source #{source_file} for #{meth_name} in #{@file_name}"
+        end
+      else
+        file_content = @content
+      end
+
+      body = find_body class_name, function, meth_obj, file_content
+
+      if body and meth_obj.document_self then
+        meth_obj.params = if p_count < -1 then # -2 is Array
+                            '(*args)'
+                          elsif p_count == -1 then # argc, argv
+                            rb_scan_args body
+                          else
+                            "(#{(1..p_count).map { |i| "p#{i}" }.join ', '})"
+                          end
+
+
+        meth_obj.record_location @top_level
+        class_obj.add_method meth_obj
+        @stats.add_method meth_obj
+        meth_obj.visibility = :private if 'private_method' == type
+      end
+    end
+  end
+
+  ##
+  # Registers a singleton class +sclass_var+ as a singleton of +class_var+
+
+  def handle_singleton sclass_var, class_var
+    class_name = @known_classes[class_var]
+
+    @known_classes[sclass_var]     = class_name
+    @singleton_classes[sclass_var] = class_name
+  end
+
+  ##
+  # Normalizes tabs in +body+
+
+  def handle_tab_width(body)
+    if /\t/ =~ body
+      tab_width = @options.tab_width
+      body.split(/\n/).map do |line|
+        1 while line.gsub!(/\t+/) do
+          ' ' * (tab_width * $&.length - $`.length % tab_width)
+        end && $~
+        line
+      end.join "\n"
+    else
+      body
+    end
+  end
+
+  ##
+  # Loads the variable map with the given +name+ from the RDoc::Store, if
+  # present.
+
+  def load_variable_map map_name
+    return {} unless files = @store.cache[map_name]
+    return {} unless name_map = files[@file_name]
+
+    class_map = {}
+
+    name_map.each do |variable, name|
+      next unless mod = @store.find_class_or_module(name)
+
+      class_map[variable] = if map_name == :c_class_variables then
+                              mod
+                            else
+                              name
+                            end
+      @known_classes[variable] = name
+    end
+
+    class_map
+  end
+
+  ##
+  # Look for directives in a normal comment block:
+  #
+  #   /*
+  #    * :title: My Awesome Project
+  #    */
+  #
+  # This method modifies the +comment+
+
+  def look_for_directives_in context, comment
+    @preprocess.handle comment, context do |directive, param|
+      case directive
+      when 'main' then
+        @options.main_page = param
+        ''
+      when 'title' then
+        @options.default_title = param if @options.respond_to? :default_title=
+        ''
+      end
+    end
+
+    comment
+  end
+
+  ##
+  # Extracts parameters from the +method_body+ and returns a method
+  # parameter string.  Follows 1.9.3dev's scan-arg-spec, see README.EXT
+
+  def rb_scan_args method_body
+    method_body =~ /rb_scan_args\((.*?)\)/m
+    return '(*args)' unless $1
+
+    $1.split(/,/)[2] =~ /"(.*?)"/ # format argument
+    format = $1.split(//)
+
+    lead = opt = trail = 0
+
+    if format.first =~ /\d/ then
+      lead = $&.to_i
+      format.shift
+      if format.first =~ /\d/ then
+        opt = $&.to_i
+        format.shift
+        if format.first =~ /\d/ then
+          trail = $&.to_i
+          format.shift
+          block_arg = true
+        end
+      end
+    end
+
+    if format.first == '*' and not block_arg then
+      var = true
+      format.shift
+      if format.first =~ /\d/ then
+        trail = $&.to_i
+        format.shift
+      end
+    end
+
+    if format.first == ':' then
+      hash = true
+      format.shift
+    end
+
+    if format.first == '&' then
+      block = true
+      format.shift
+    end
+
+    # if the format string is not empty there's a bug in the C code, ignore it
+
+    args = []
+    position = 1
+
+    (1...(position + lead)).each do |index|
+      args << "p#{index}"
+    end
+
+    position += lead
+
+    (position...(position + opt)).each do |index|
+      args << "p#{index} = v#{index}"
+    end
+
+    position += opt
+
+    if var then
+      args << '*args'
+      position += 1
+    end
+
+    (position...(position + trail)).each do |index|
+      args << "p#{index}"
+    end
+
+    position += trail
+
+    if hash then
+      args << "p#{position} = {}"
+    end
+
+    args << '&block' if block
+
+    "(#{args.join ', '})"
+  end
+
+  ##
+  # Removes lines that are commented out that might otherwise get picked up
+  # when scanning for classes and methods
+
+  def remove_commented_out_lines
+    @content.gsub!(%r%//.*rb_define_%, '//')
+  end
+
+  ##
+  # Extracts the classes, modules, methods, attributes, constants and aliases
+  # from a C file and returns an RDoc::TopLevel for this file
+
+  def scan
+    remove_commented_out_lines
+
+    do_modules
+    do_classes
+    do_missing
+
+    do_constants
+    do_methods
+    do_includes
+    do_aliases
+    do_attrs
+
+    deduplicate_call_seq
+
+    @store.add_c_variables self
+
+    @top_level
+  end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/changelog.rb b/jni/ruby/lib/rdoc/parser/changelog.rb
new file mode 100644
index 0000000..a3567c1
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/changelog.rb
@@ -0,0 +1,198 @@
+require 'time'
+
+##
+# A ChangeLog file parser.
+#
+# This parser converts a ChangeLog into an RDoc::Markup::Document.  When
+# viewed as HTML a ChangeLog page will have an entry for each day's entries in
+# the sidebar table of contents.
+#
+# This parser is meant to parse the MRI ChangeLog, but can be used to parse any
+# {GNU style Change
+# Log}[http://www.gnu.org/prep/standards/html_node/Style-of-Change-Logs.html].
+
+class RDoc::Parser::ChangeLog < RDoc::Parser
+
+  include RDoc::Parser::Text
+
+  parse_files_matching(/(\/|\\|\A)ChangeLog[^\/\\]*\z/)
+
+  ##
+  # Attaches the +continuation+ of the previous line to the +entry_body+.
+  #
+  # Continued function listings are joined together as a single entry.
+  # Continued descriptions are joined to make a single paragraph.
+
+  def continue_entry_body entry_body, continuation
+    return unless last = entry_body.last
+
+    if last =~ /\)\s*\z/ and continuation =~ /\A\(/ then
+      last.sub!(/\)\s*\z/, ',')
+      continuation.sub!(/\A\(/, '')
+    end
+
+    if last =~ /\s\z/ then
+      last << continuation
+    else
+      last << ' ' << continuation
+    end
+  end
+
+  ##
+  # Creates an RDoc::Markup::Document given the +groups+ of ChangeLog entries.
+
+  def create_document groups
+    doc = RDoc::Markup::Document.new
+    doc.omit_headings_below = 2
+    doc.file = @top_level
+
+    doc << RDoc::Markup::Heading.new(1, File.basename(@file_name))
+    doc << RDoc::Markup::BlankLine.new
+
+    groups.sort_by do |day,| day end.reverse_each do |day, entries|
+      doc << RDoc::Markup::Heading.new(2, day.dup)
+      doc << RDoc::Markup::BlankLine.new
+
+      doc.concat create_entries entries
+    end
+
+    doc
+  end
+
+  ##
+  # Returns a list of ChangeLog entries an RDoc::Markup nodes for the given
+  # +entries+.
+
+  def create_entries entries
+    out = []
+
+    entries.each do |entry, items|
+      out << RDoc::Markup::Heading.new(3, entry)
+      out << RDoc::Markup::BlankLine.new
+
+      out << create_items(items)
+    end
+
+    out
+  end
+
+  ##
+  # Returns an RDoc::Markup::List containing the given +items+ in the
+  # ChangeLog
+
+  def create_items items
+    list = RDoc::Markup::List.new :NOTE
+
+    items.each do |item|
+      item =~ /\A(.*?(?:\([^)]+\))?):\s*/
+
+      title = $1
+      body = $'
+
+      paragraph = RDoc::Markup::Paragraph.new body
+      list_item = RDoc::Markup::ListItem.new title, paragraph
+      list << list_item
+    end
+
+    list
+  end
+
+  ##
+  # Groups +entries+ by date.
+
+  def group_entries entries
+    entries.group_by do |title, _|
+      begin
+        Time.parse(title).strftime '%Y-%m-%d'
+      rescue NoMethodError, ArgumentError
+        time, = title.split '  ', 2
+        Time.parse(time).strftime '%Y-%m-%d'
+      end
+    end
+  end
+
+  ##
+  # Parses the entries in the ChangeLog.
+  #
+  # Returns an Array of each ChangeLog entry in order of parsing.
+  #
+  # A ChangeLog entry is an Array containing the ChangeLog title (date and
+  # committer) and an Array of ChangeLog items (file and function changed with
+  # description).
+  #
+  # An example result would be:
+  #
+  #    [ 'Tue Dec  4 08:33:46 2012  Eric Hodel  <drbrain@segment7.net>',
+  #      [ 'README.EXT:  Converted to RDoc format',
+  #        'README.EXT.ja:  ditto']]
+
+  def parse_entries
+    entries = []
+    entry_name = nil
+    entry_body = []
+
+    @content.each_line do |line|
+      case line
+      when /^\s*$/ then
+        next
+      when /^\w.*/ then
+        entries << [entry_name, entry_body] if entry_name
+
+        entry_name = $&
+
+        begin
+          time = Time.parse entry_name
+          # HACK Ruby 1.8 does not raise ArgumentError for Time.parse "Other"
+          entry_name = nil unless entry_name =~ /#{time.year}/
+        rescue NoMethodError
+          # HACK Ruby 2.1.2 and earlier raises NoMethodError if time part is absent
+          entry_name.split '  ', 2
+        rescue ArgumentError
+          if /out of range/ =~ $!.message
+            Time.parse(entry_name.split('  ', 2)[0]) rescue entry_name = nil
+          else
+            entry_name = nil
+          end
+        end
+
+        entry_body = []
+      when /^(\t| {8})?\*\s*(.*)/ then # "\t* file.c (func): ..."
+        entry_body << $2
+      when /^(\t| {8})?\s*(\(.*)/ then # "\t(func): ..."
+        entry = $2
+
+        if entry_body.last =~ /:/ then
+          entry_body << entry
+        else
+          continue_entry_body entry_body, entry
+        end
+      when /^(\t| {8})?\s*(.*)/ then
+        continue_entry_body entry_body, $2
+      end
+    end
+
+    entries << [entry_name, entry_body] if entry_name
+
+    entries.reject! do |(entry,_)|
+      entry == nil
+    end
+
+    entries
+  end
+
+  ##
+  # Converts the ChangeLog into an RDoc::Markup::Document
+
+  def scan
+    entries = parse_entries
+    grouped_entries = group_entries entries
+
+    doc = create_document grouped_entries
+
+    @top_level.comment = doc
+
+    @top_level
+  end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/markdown.rb b/jni/ruby/lib/rdoc/parser/markdown.rb
new file mode 100644
index 0000000..6fd88cf
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/markdown.rb
@@ -0,0 +1,23 @@
+##
+# Parse a Markdown format file.  The parsed RDoc::Markup::Document is attached
+# as a file comment.
+
+class RDoc::Parser::Markdown < RDoc::Parser
+
+  include RDoc::Parser::Text
+
+  parse_files_matching(/\.(md|markdown)(?:\.[^.]+)?$/)
+
+  ##
+  # Creates an Markdown-format TopLevel for the given file.
+
+  def scan
+    comment = RDoc::Comment.new @content, @top_level
+    comment.format = 'markdown'
+
+    @top_level.comment = comment
+  end
+
+end
+
+
diff --git a/jni/ruby/lib/rdoc/parser/rd.rb b/jni/ruby/lib/rdoc/parser/rd.rb
new file mode 100644
index 0000000..09069ae
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/rd.rb
@@ -0,0 +1,22 @@
+##
+# Parse a RD format file.  The parsed RDoc::Markup::Document is attached as a
+# file comment.
+
+class RDoc::Parser::RD < RDoc::Parser
+
+  include RDoc::Parser::Text
+
+  parse_files_matching(/\.rd(?:\.[^.]+)?$/)
+
+  ##
+  # Creates an rd-format TopLevel for the given file.
+
+  def scan
+    comment = RDoc::Comment.new @content, @top_level
+    comment.format = 'rd'
+
+    @top_level.comment = comment
+  end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/ruby.rb b/jni/ruby/lib/rdoc/parser/ruby.rb
new file mode 100644
index 0000000..ce1083e
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/ruby.rb
@@ -0,0 +1,2160 @@
+##
+# This file contains stuff stolen outright from:
+#
+#   rtags.rb -
+#   ruby-lex.rb - ruby lexcal analyzer
+#   ruby-token.rb - ruby tokens
+#       by Keiju ISHITSUKA (Nippon Rational Inc.)
+#
+
+$TOKEN_DEBUG ||= nil
+
+##
+# Extracts code elements from a source file returning a TopLevel object
+# containing the constituent file elements.
+#
+# This file is based on rtags
+#
+# RubyParser understands how to document:
+# * classes
+# * modules
+# * methods
+# * constants
+# * aliases
+# * private, public, protected
+# * private_class_function, public_class_function
+# * module_function
+# * attr, attr_reader, attr_writer, attr_accessor
+# * extra accessors given on the command line
+# * metaprogrammed methods
+# * require
+# * include
+#
+# == Method Arguments
+#
+#--
+# NOTE: I don't think this works, needs tests, remove the paragraph following
+# this block when known to work
+#
+# The parser extracts the arguments from the method definition.  You can
+# override this with a custom argument definition using the :args: directive:
+#
+#   ##
+#   # This method tries over and over until it is tired
+#
+#   def go_go_go(thing_to_try, tries = 10) # :args: thing_to_try
+#     puts thing_to_try
+#     go_go_go thing_to_try, tries - 1
+#   end
+#
+# If you have a more-complex set of overrides you can use the :call-seq:
+# directive:
+#++
+#
+# The parser extracts the arguments from the method definition.  You can
+# override this with a custom argument definition using the :call-seq:
+# directive:
+#
+#   ##
+#   # This method can be called with a range or an offset and length
+#   #
+#   # :call-seq:
+#   #   my_method(Range)
+#   #   my_method(offset, length)
+#
+#   def my_method(*args)
+#   end
+#
+# The parser extracts +yield+ expressions from method bodies to gather the
+# yielded argument names.  If your method manually calls a block instead of
+# yielding or you want to override the discovered argument names use
+# the :yields: directive:
+#
+#   ##
+#   # My method is awesome
+#
+#   def my_method(&block) # :yields: happy, times
+#     block.call 1, 2
+#   end
+#
+# == Metaprogrammed Methods
+#
+# To pick up a metaprogrammed method, the parser looks for a comment starting
+# with '##' before an identifier:
+#
+#   ##
+#   # This is a meta-programmed method!
+#
+#   add_my_method :meta_method, :arg1, :arg2
+#
+# The parser looks at the token after the identifier to determine the name, in
+# this example, :meta_method.  If a name cannot be found, a warning is printed
+# and 'unknown is used.
+#
+# You can force the name of a method using the :method: directive:
+#
+#   ##
+#   # :method: some_method!
+#
+# By default, meta-methods are instance methods.  To indicate that a method is
+# a singleton method instead use the :singleton-method: directive:
+#
+#   ##
+#   # :singleton-method:
+#
+# You can also use the :singleton-method: directive with a name:
+#
+#   ##
+#   # :singleton-method: some_method!
+#
+# You can define arguments for metaprogrammed methods via either the
+# :call-seq:, :arg: or :args: directives.
+#
+# Additionally you can mark a method as an attribute by
+# using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:.  Just like
+# for :method:, the name is optional.
+#
+#   ##
+#   # :attr_reader: my_attr_name
+#
+# == Hidden methods and attributes
+#
+# You can provide documentation for methods that don't appear using
+# the :method:, :singleton-method: and :attr: directives:
+#
+#   ##
+#   # :attr_writer: ghost_writer
+#   # There is an attribute here, but you can't see it!
+#
+#   ##
+#   # :method: ghost_method
+#   # There is a method here, but you can't see it!
+#
+#   ##
+#   # this is a comment for a regular method
+#
+#   def regular_method() end
+#
+# Note that by default, the :method: directive will be ignored if there is a
+# standard rdocable item following it.
+
+class RDoc::Parser::Ruby < RDoc::Parser
+
+  parse_files_matching(/\.rbw?$/)
+
+  include RDoc::RubyToken
+  include RDoc::TokenStream
+  include RDoc::Parser::RubyTools
+
+  ##
+  # RDoc::NormalClass type
+
+  NORMAL = "::"
+
+  ##
+  # RDoc::SingleClass type
+
+  SINGLE = "<<"
+
+  ##
+  # Creates a new Ruby parser.
+
+  def initialize(top_level, file_name, content, options, stats)
+    super
+
+    @size = 0
+    @token_listeners = nil
+    @scanner = RDoc::RubyLex.new content, @options
+    @scanner.exception_on_syntax_error = false
+    @prev_seek = nil
+    @markup = @options.markup
+    @track_visibility = :nodoc != @options.visibility
+
+    @encoding = nil
+    @encoding = @options.encoding if Object.const_defined? :Encoding
+
+    reset
+  end
+
+  ##
+  # Retrieves the read token stream and replaces +pattern+ with +replacement+
+  # using gsub.  If the result is only a ";" returns an empty string.
+
+  def get_tkread_clean pattern, replacement # :nodoc:
+    read = get_tkread.gsub(pattern, replacement).strip
+    return '' if read == ';'
+    read
+  end
+
+  ##
+  # Extracts the visibility information for the visibility token +tk+
+  # and +single+ class type identifier.
+  #
+  # Returns the visibility type (a string), the visibility (a symbol) and
+  # +singleton+ if the methods following should be converted to singleton
+  # methods.
+
+  def get_visibility_information tk, single # :nodoc:
+    vis_type  = tk.name
+    singleton = single == SINGLE
+
+    vis =
+      case vis_type
+      when 'private'   then :private
+      when 'protected' then :protected
+      when 'public'    then :public
+      when 'private_class_method' then
+        singleton = true
+        :private
+      when 'public_class_method' then
+        singleton = true
+        :public
+      when 'module_function' then
+        singleton = true
+        :public
+      else
+        raise RDoc::Error, "Invalid visibility: #{tk.name}"
+      end
+
+    return vis_type, vis, singleton
+  end
+
+  ##
+  # Look for the first comment in a file that isn't a shebang line.
+
+  def collect_first_comment
+    skip_tkspace
+    comment = ''
+    comment.force_encoding @encoding if @encoding
+    first_line = true
+    first_comment_tk_class = nil
+
+    tk = get_tk
+
+    while TkCOMMENT === tk
+      if first_line and tk.text =~ /\A#!/ then
+        skip_tkspace
+        tk = get_tk
+      elsif first_line and tk.text =~ /\A#\s*-\*-/ then
+        first_line = false
+        skip_tkspace
+        tk = get_tk
+      else
+        break if first_comment_tk_class and not first_comment_tk_class === tk
+        first_comment_tk_class = tk.class
+
+        first_line = false
+        comment << tk.text << "\n"
+        tk = get_tk
+
+        if TkNL === tk then
+          skip_tkspace false
+          tk = get_tk
+        end
+      end
+    end
+
+    unget_tk tk
+
+    new_comment comment
+  end
+
+  ##
+  # Consumes trailing whitespace from the token stream
+
+  def consume_trailing_spaces # :nodoc:
+    get_tkread
+    skip_tkspace false
+  end
+
+  ##
+  # Creates a new attribute in +container+ with +name+.
+
+  def create_attr container, single, name, rw, comment # :nodoc:
+    att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+    record_location att
+
+    container.add_attribute att
+    @stats.add_attribute att
+
+    att
+  end
+
+  ##
+  # Creates a module alias in +container+ at +rhs_name+ (or at the top-level
+  # for "::") with the name from +constant+.
+
+  def create_module_alias container, constant, rhs_name # :nodoc:
+    mod = if rhs_name =~ /^::/ then
+            @store.find_class_or_module rhs_name
+          else
+            container.find_module_named rhs_name
+          end
+
+    container.add_module_alias mod, constant.name, @top_level if mod
+  end
+
+  ##
+  # Aborts with +msg+
+
+  def error(msg)
+    msg = make_message msg
+
+    abort msg
+  end
+
+  ##
+  # Looks for a true or false token.  Returns false if TkFALSE or TkNIL are
+  # found.
+
+  def get_bool
+    skip_tkspace
+    tk = get_tk
+    case tk
+    when TkTRUE
+      true
+    when TkFALSE, TkNIL
+      false
+    else
+      unget_tk tk
+      true
+    end
+  end
+
+  ##
+  # Look for the name of a class of module (optionally with a leading :: or
+  # with :: separated named) and return the ultimate name, the associated
+  # container, and the given name (with the ::).
+
+  def get_class_or_module container, ignore_constants = false
+    skip_tkspace
+    name_t = get_tk
+    given_name = ''
+
+    # class ::A -> A is in the top level
+    case name_t
+    when TkCOLON2, TkCOLON3 then # bug
+      name_t = get_tk
+      container = @top_level
+      given_name << '::'
+    end
+
+    skip_tkspace false
+    given_name << name_t.name
+
+    while TkCOLON2 === peek_tk do
+      prev_container = container
+      container = container.find_module_named name_t.name
+      container ||=
+        if ignore_constants then
+          RDoc::Context.new
+        else
+          c = prev_container.add_module RDoc::NormalModule, name_t.name
+          c.ignore unless prev_container.document_children
+          @top_level.add_to_classes_or_modules c
+          c
+        end
+
+      record_location container
+
+      get_tk
+      skip_tkspace false
+      name_t = get_tk
+      given_name << '::' << name_t.name
+    end
+
+    skip_tkspace false
+
+    return [container, name_t, given_name]
+  end
+
+  ##
+  # Return a superclass, which can be either a constant of an expression
+
+  def get_class_specification
+    case peek_tk
+    when TkSELF then return 'self'
+    when TkGVAR then return ''
+    end
+
+    res = get_constant
+
+    skip_tkspace false
+
+    get_tkread # empty out read buffer
+
+    tk = get_tk
+
+    case tk
+    when TkNL, TkCOMMENT, TkSEMICOLON then
+      unget_tk(tk)
+      return res
+    end
+
+    res += parse_call_parameters(tk)
+    res
+  end
+
+  ##
+  # Parse a constant, which might be qualified by one or more class or module
+  # names
+
+  def get_constant
+    res = ""
+    skip_tkspace false
+    tk = get_tk
+
+    while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
+      res += tk.name
+      tk = get_tk
+    end
+
+    unget_tk(tk)
+    res
+  end
+
+  ##
+  # Get a constant that may be surrounded by parens
+
+  def get_constant_with_optional_parens
+    skip_tkspace false
+
+    nest = 0
+
+    while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do
+      get_tk
+      skip_tkspace
+      nest += 1
+    end
+
+    name = get_constant
+
+    while nest > 0
+      skip_tkspace
+      tk = get_tk
+      nest -= 1 if TkRPAREN === tk
+    end
+
+    name
+  end
+
+  ##
+  # Little hack going on here. In the statement:
+  #
+  #   f = 2*(1+yield)
+  #
+  # We see the RPAREN as the next token, so we need to exit early.  This still
+  # won't catch all cases (such as "a = yield + 1"
+
+  def get_end_token tk # :nodoc:
+    case tk
+    when TkLPAREN, TkfLPAREN
+      TkRPAREN
+    when TkRPAREN
+      nil
+    else
+      TkNL
+    end
+  end
+
+  ##
+  # Retrieves the method container for a singleton method.
+
+  def get_method_container container, name_t # :nodoc:
+    prev_container = container
+    container = container.find_module_named(name_t.name)
+
+    unless container then
+      constant = prev_container.constants.find do |const|
+        const.name == name_t.name
+      end
+
+      if constant then
+        parse_method_dummy prev_container
+        return
+      end
+    end
+
+    unless container then
+      # TODO seems broken, should starting at Object in @store
+      obj = name_t.name.split("::").inject(Object) do |state, item|
+        state.const_get(item)
+      end rescue nil
+
+      type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
+
+      unless [Class, Module].include?(obj.class) then
+        warn("Couldn't find #{name_t.name}. Assuming it's a module")
+      end
+
+      if type == RDoc::NormalClass then
+        sclass = obj.superclass ? obj.superclass.name : nil
+        container = prev_container.add_class type, name_t.name, sclass
+      else
+        container = prev_container.add_module type, name_t.name
+      end
+
+      record_location container
+    end
+
+    container
+  end
+
+  ##
+  # Extracts a name or symbol from the token stream.
+
+  def get_symbol_or_name
+    tk = get_tk
+    case tk
+    when TkSYMBOL then
+      text = tk.text.sub(/^:/, '')
+
+      if TkASSIGN === peek_tk then
+        get_tk
+        text << '='
+      end
+
+      text
+    when TkId, TkOp then
+      tk.name
+    when TkAMPER,
+         TkDSTRING,
+         TkSTAR,
+         TkSTRING then
+      tk.text
+    else
+      raise RDoc::Error, "Name or symbol expected (got #{tk})"
+    end
+  end
+
+  def stop_at_EXPR_END # :nodoc:
+    @scanner.lex_state == :EXPR_END || !@scanner.continue
+  end
+
+  ##
+  # Marks containers between +container+ and +ancestor+ as ignored
+
+  def suppress_parents container, ancestor # :nodoc:
+    while container and container != ancestor do
+      container.suppress unless container.documented?
+      container = container.parent
+    end
+  end
+
+  ##
+  # Look for directives in a normal comment block:
+  #
+  #   # :stopdoc:
+  #   # Don't display comment from this point forward
+  #
+  # This routine modifies its +comment+ parameter.
+
+  def look_for_directives_in context, comment
+    @preprocess.handle comment, context do |directive, param|
+      case directive
+      when 'method', 'singleton-method',
+           'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
+        false # handled elsewhere
+      when 'section' then
+        context.set_current_section param, comment.dup
+        comment.text = ''
+        break
+      end
+    end
+
+    remove_private_comments comment
+  end
+
+  ##
+  # Adds useful info about the parser to +message+
+
+  def make_message message
+    prefix = "#{@file_name}:"
+
+    prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner
+
+    "#{prefix} #{message}"
+  end
+
+  ##
+  # Creates a comment with the correct format
+
+  def new_comment comment
+    c = RDoc::Comment.new comment, @top_level
+    c.format = @markup
+    c
+  end
+
+  ##
+  # Creates an RDoc::Attr for the name following +tk+, setting the comment to
+  # +comment+.
+
+  def parse_attr(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    args = parse_symbol_arg 1
+    if args.size > 0 then
+      name = args[0]
+      rw = "R"
+      skip_tkspace false
+      tk = get_tk
+
+      if TkCOMMA === tk then
+        rw = "RW" if get_bool
+      else
+        unget_tk tk
+      end
+
+      att = create_attr context, single, name, rw, comment
+      att.offset = offset
+      att.line   = line_no
+
+      read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
+    else
+      warn "'attr' ignored - looks like a variable"
+    end
+  end
+
+  ##
+  # Creates an RDoc::Attr for each attribute listed after +tk+, setting the
+  # comment for each to +comment+.
+
+  def parse_attr_accessor(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    args = parse_symbol_arg
+    rw = "?"
+
+    tmp = RDoc::CodeObject.new
+    read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
+    # TODO In most other places we let the context keep track of document_self
+    # and add found items appropriately but here we do not.  I'm not sure why.
+    return if @track_visibility and not tmp.document_self
+
+    case tk.name
+    when "attr_reader"   then rw = "R"
+    when "attr_writer"   then rw = "W"
+    when "attr_accessor" then rw = "RW"
+    else
+      rw = '?'
+    end
+
+    for name in args
+      att = create_attr context, single, name, rw, comment
+      att.offset = offset
+      att.line   = line_no
+    end
+  end
+
+  ##
+  # Parses an +alias+ in +context+ with +comment+
+
+  def parse_alias(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    skip_tkspace
+
+    if TkLPAREN === peek_tk then
+      get_tk
+      skip_tkspace
+    end
+
+    new_name = get_symbol_or_name
+
+    @scanner.lex_state = :EXPR_FNAME
+
+    skip_tkspace
+    if TkCOMMA === peek_tk then
+      get_tk
+      skip_tkspace
+    end
+
+    begin
+      old_name = get_symbol_or_name
+    rescue RDoc::Error
+      return
+    end
+
+    al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
+                         single == SINGLE)
+    record_location al
+    al.offset = offset
+    al.line   = line_no
+
+    read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
+    context.add_alias al
+    @stats.add_alias al
+
+    al
+  end
+
+  ##
+  # Extracts call parameters from the token stream.
+
+  def parse_call_parameters(tk)
+    end_token = case tk
+                when TkLPAREN, TkfLPAREN
+                  TkRPAREN
+                when TkRPAREN
+                  return ""
+                else
+                  TkNL
+                end
+    nest = 0
+
+    loop do
+      case tk
+      when TkSEMICOLON
+        break
+      when TkLPAREN, TkfLPAREN
+        nest += 1
+      when end_token
+        if end_token == TkRPAREN
+          nest -= 1
+          break if @scanner.lex_state == :EXPR_END and nest <= 0
+        else
+          break unless @scanner.continue
+        end
+      when TkCOMMENT, TkASSIGN, TkOPASGN
+        unget_tk(tk)
+        break
+      when nil then
+        break
+      end
+      tk = get_tk
+    end
+
+    get_tkread_clean "\n", " "
+  end
+
+  ##
+  # Parses a class in +context+ with +comment+
+
+  def parse_class container, single, tk, comment
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    declaration_context = container
+    container, name_t, given_name = get_class_or_module container
+
+    cls =
+      case name_t
+      when TkCONSTANT
+        parse_class_regular container, declaration_context, single,
+          name_t, given_name, comment
+      when TkLSHFT
+        case name = get_class_specification
+        when 'self', container.name
+          parse_statements container, SINGLE
+          return # don't update offset or line
+        else
+          parse_class_singleton container, name, comment
+        end
+      else
+        warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}"
+        return
+      end
+
+    cls.offset = offset
+    cls.line   = line_no
+
+    cls
+  end
+
+  ##
+  # Parses and creates a regular class
+
+  def parse_class_regular container, declaration_context, single, # :nodoc:
+                          name_t, given_name, comment
+    superclass = '::Object'
+
+    if given_name =~ /^::/ then
+      declaration_context = @top_level
+      given_name = $'
+    end
+
+    if TkLT === peek_tk then
+      get_tk
+      skip_tkspace
+      superclass = get_class_specification
+      superclass = '(unknown)' if superclass.empty?
+    end
+
+    cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
+    cls = declaration_context.add_class cls_type, given_name, superclass
+    cls.ignore unless container.document_children
+
+    read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+    record_location cls
+
+    cls.add_comment comment, @top_level
+
+    @top_level.add_to_classes_or_modules cls
+    @stats.add_class cls
+
+    suppress_parents container, declaration_context unless cls.document_self
+
+    parse_statements cls
+
+    cls
+  end
+
+  ##
+  # Parses a singleton class in +container+ with the given +name+ and
+  # +comment+.
+
+  def parse_class_singleton container, name, comment # :nodoc:
+    other = @store.find_class_named name
+
+    unless other then
+      if name =~ /^::/ then
+        name = $'
+        container = @top_level
+      end
+
+      other = container.add_module RDoc::NormalModule, name
+      record_location other
+
+      # class << $gvar
+      other.ignore if name.empty?
+
+      other.add_comment comment, @top_level
+    end
+
+    # notify :nodoc: all if not a constant-named class/module
+    # (and remove any comment)
+    unless name =~ /\A(::)?[A-Z]/ then
+      other.document_self = nil
+      other.document_children = false
+      other.clear_comment
+    end
+
+    @top_level.add_to_classes_or_modules other
+    @stats.add_class other
+
+    read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
+    parse_statements(other, SINGLE)
+
+    other
+  end
+
+  ##
+  # Parses a constant in +context+ with +comment+.  If +ignore_constants+ is
+  # true, no found constants will be added to RDoc.
+
+  def parse_constant container, tk, comment, ignore_constants = false
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    name = tk.name
+    skip_tkspace false
+
+    return unless name =~ /^\w+$/
+
+    eq_tk = get_tk
+
+    if TkCOLON2 === eq_tk then
+      unget_tk eq_tk
+      unget_tk tk
+
+      container, name_t, = get_class_or_module container, ignore_constants
+
+      name = name_t.name
+
+      eq_tk = get_tk
+    end
+
+    unless TkASSIGN === eq_tk then
+      unget_tk eq_tk
+      return false
+    end
+
+    if TkGT === peek_tk then
+      unget_tk eq_tk
+      return
+    end
+
+    value = ''
+    con = RDoc::Constant.new name, value, comment
+
+    body = parse_constant_body container, con
+
+    return unless body
+
+    value.replace body
+    record_location con
+    con.offset = offset
+    con.line   = line_no
+    read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
+
+    @stats.add_constant con
+    container.add_constant con
+
+    true
+  end
+
+  def parse_constant_body container, constant # :nodoc:
+    nest     = 0
+    rhs_name = ''
+
+    get_tkread
+
+    tk = get_tk
+
+    loop do
+      case tk
+      when TkSEMICOLON then
+        break if nest <= 0
+      when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK,
+           TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then
+        nest += 1
+      when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
+        nest -= 1
+      when TkCOMMENT then
+        if nest <= 0 and stop_at_EXPR_END then
+          unget_tk tk
+          break
+        else
+          unget_tk tk
+          read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
+        end
+      when TkCONSTANT then
+        rhs_name << tk.name
+
+        if nest <= 0 and TkNL === peek_tk then
+          create_module_alias container, constant, rhs_name
+          break
+        end
+      when TkNL then
+        if nest <= 0 and stop_at_EXPR_END then
+          unget_tk tk
+          break
+        end
+      when TkCOLON2, TkCOLON3 then
+        rhs_name << '::'
+      when nil then
+        break
+      end
+      tk = get_tk
+    end
+
+    get_tkread_clean(/^[ \t]+/, '')
+  end
+
+  ##
+  # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for
+  # :method: or :attr: directives in +comment+.
+
+  def parse_comment container, tk, comment
+    return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
+    column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    text = comment.text
+
+    singleton = !!text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+
+    co =
+      if text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+        parse_comment_ghost container, text, $1, column, line_no, comment
+      elsif text.sub!(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+        parse_comment_attr container, $1, $3, comment
+      end
+
+    if co then
+      co.singleton = singleton
+      co.offset    = offset
+      co.line      = line_no
+    end
+
+    true
+  end
+
+  ##
+  # Parse a comment that is describing an attribute in +container+ with the
+  # given +name+ and +comment+.
+
+  def parse_comment_attr container, type, name, comment # :nodoc:
+    return if name.empty?
+
+    rw = case type
+         when 'attr_reader' then 'R'
+         when 'attr_writer' then 'W'
+         else 'RW'
+         end
+
+    create_attr container, NORMAL, name, rw, comment
+  end
+
+  def parse_comment_ghost container, text, name, column, line_no, # :nodoc:
+                          comment
+    name = nil if name.empty?
+
+    meth = RDoc::GhostMethod.new get_tkread, name
+    record_location meth
+
+    meth.start_collecting_tokens
+    indent = TkSPACE.new 0, 1, 1
+    indent.set_text " " * column
+
+    position_comment = TkCOMMENT.new 0, line_no, 1
+    position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
+    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+
+    meth.params =
+      if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then
+        $1
+      else
+        ''
+      end
+
+    comment.normalize
+    comment.extract_call_seq meth
+
+    return unless meth.name
+
+    container.add_method meth
+
+    meth.comment = comment
+
+    @stats.add_method meth
+
+    meth
+  end
+
+  ##
+  # Creates an RDoc::Method on +container+ from +comment+ if there is a
+  # Signature section in the comment
+
+  def parse_comment_tomdoc container, tk, comment
+    return unless signature = RDoc::TomDoc.signature(comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    name, = signature.split %r%[ \(]%, 2
+
+    meth = RDoc::GhostMethod.new get_tkread, name
+    record_location meth
+    meth.offset    = offset
+    meth.line      = line_no
+
+    meth.start_collecting_tokens
+    indent = TkSPACE.new 0, 1, 1
+    indent.set_text " " * offset
+
+    position_comment = TkCOMMENT.new 0, line_no, 1
+    position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
+    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+
+    meth.call_seq = signature
+
+    comment.normalize
+
+    return unless meth.name
+
+    container.add_method meth
+
+    meth.comment = comment
+
+    @stats.add_method meth
+  end
+
+  ##
+  # Parses an +include+ or +extend+, indicated by the +klass+ and adds it to
+  # +container+ # with +comment+
+
+  def parse_extend_or_include klass, container, comment # :nodoc:
+    loop do
+      skip_tkspace_comment
+
+      name = get_constant_with_optional_parens
+
+      unless name.empty? then
+        obj = container.add klass, name, comment
+        record_location obj
+      end
+
+      return unless TkCOMMA === peek_tk
+
+      get_tk
+    end
+  end
+
+  ##
+  # Parses identifiers that can create new methods or change visibility.
+  #
+  # Returns true if the comment was not consumed.
+
+  def parse_identifier container, single, tk, comment # :nodoc:
+    case tk.name
+    when 'private', 'protected', 'public', 'private_class_method',
+         'public_class_method', 'module_function' then
+      parse_visibility container, single, tk
+      return true
+    when 'attr' then
+      parse_attr container, single, tk, comment
+    when /^attr_(reader|writer|accessor)$/ then
+      parse_attr_accessor container, single, tk, comment
+    when 'alias_method' then
+      parse_alias container, single, tk, comment
+    when 'require', 'include' then
+      # ignore
+    else
+      if comment.text =~ /\A#\#$/ then
+        case comment.text
+        when /^# +:?attr(_reader|_writer|_accessor)?:/ then
+          parse_meta_attr container, single, tk, comment
+        else
+          method = parse_meta_method container, single, tk, comment
+          method.params = container.params if
+            container.params
+          method.block_params = container.block_params if
+            container.block_params
+        end
+      end
+    end
+
+    false
+  end
+
+  ##
+  # Parses a meta-programmed attribute and creates an RDoc::Attr.
+  #
+  # To create foo and bar attributes on class C with comment "My attributes":
+  #
+  #   class C
+  #
+  #     ##
+  #     # :attr:
+  #     #
+  #     # My attributes
+  #
+  #     my_attr :foo, :bar
+  #
+  #   end
+  #
+  # To create a foo attribute on class C with comment "My attribute":
+  #
+  #   class C
+  #
+  #     ##
+  #     # :attr: foo
+  #     #
+  #     # My attribute
+  #
+  #     my_attr :foo, :bar
+  #
+  #   end
+
+  def parse_meta_attr(context, single, tk, comment)
+    args = parse_symbol_arg
+    rw = "?"
+
+    # If nodoc is given, don't document any of them
+
+    tmp = RDoc::CodeObject.new
+    read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
+
+    if comment.text.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+      rw = case $1
+           when 'attr_reader' then 'R'
+           when 'attr_writer' then 'W'
+           else 'RW'
+           end
+      name = $3 unless $3.empty?
+    end
+
+    if name then
+      att = create_attr context, single, name, rw, comment
+    else
+      args.each do |attr_name|
+        att = create_attr context, single, attr_name, rw, comment
+      end
+    end
+
+    att
+  end
+
+  ##
+  # Parses a meta-programmed method
+
+  def parse_meta_method(container, single, tk, comment)
+    column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    start_collecting_tokens
+    add_token tk
+    add_token_listener self
+
+    skip_tkspace false
+
+    singleton = !!comment.text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+
+    name = parse_meta_method_name comment, tk
+
+    return unless name
+
+    meth = RDoc::MetaMethod.new get_tkread, name
+    record_location meth
+    meth.offset = offset
+    meth.line   = line_no
+    meth.singleton = singleton
+
+    remove_token_listener self
+
+    meth.start_collecting_tokens
+    indent = TkSPACE.new 0, 1, 1
+    indent.set_text " " * column
+
+    position_comment = TkCOMMENT.new 0, line_no, 1
+    position_comment.value = "# File #{@top_level.relative_name}, line #{line_no}"
+    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+    meth.add_tokens @token_stream
+
+    parse_meta_method_params container, single, meth, tk, comment
+
+    meth.comment = comment
+
+    @stats.add_method meth
+
+    meth
+  end
+
+  ##
+  # Parses the name of a metaprogrammed method.  +comment+ is used to
+  # determine the name while +tk+ is used in an error message if the name
+  # cannot be determined.
+
+  def parse_meta_method_name comment, tk # :nodoc:
+    if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+      return $1 unless $1.empty?
+    end
+
+    name_t = get_tk
+
+    case name_t
+    when TkSYMBOL then
+      name_t.text[1..-1]
+    when TkSTRING then
+      name_t.value[1..-2]
+    when TkASSIGN then # ignore
+      remove_token_listener self
+
+      nil
+    else
+      warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'"
+      'unknown'
+    end
+  end
+
+  ##
+  # Parses the parameters and block for a meta-programmed method.
+
+  def parse_meta_method_params container, single, meth, tk, comment # :nodoc:
+    token_listener meth do
+      meth.params = ''
+
+      comment.normalize
+      comment.extract_call_seq meth
+
+      container.add_method meth
+
+      last_tk = tk
+
+      while tk = get_tk do
+        case tk
+        when TkSEMICOLON then
+          break
+        when TkNL then
+          break unless last_tk and TkCOMMA === last_tk
+        when TkSPACE then
+          # expression continues
+        when TkDO then
+          parse_statements container, single, meth
+          break
+        else
+          last_tk = tk
+        end
+      end
+    end
+  end
+
+  ##
+  # Parses a normal method defined by +def+
+
+  def parse_method(container, single, tk, comment)
+    singleton = nil
+    added_container = false
+    name = nil
+    column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    start_collecting_tokens
+    add_token tk
+
+    token_listener self do
+      prev_container = container
+      name, container, singleton = parse_method_name container
+      added_container = container != prev_container
+    end
+
+    return unless name
+
+    meth = RDoc::AnyMethod.new get_tkread, name
+    meth.singleton = single == SINGLE ? true : singleton
+
+    record_location meth
+    meth.offset = offset
+    meth.line   = line_no
+
+    meth.start_collecting_tokens
+    indent = TkSPACE.new 0, 1, 1
+    indent.set_text " " * column
+
+    token = TkCOMMENT.new 0, line_no, 1
+    token.set_text "# File #{@top_level.relative_name}, line #{line_no}"
+    meth.add_tokens [token, NEWLINE_TOKEN, indent]
+    meth.add_tokens @token_stream
+
+    parse_method_params_and_body container, single, meth, added_container
+
+    comment.normalize
+    comment.extract_call_seq meth
+
+    meth.comment = comment
+
+    @stats.add_method meth
+  end
+
+  ##
+  # Parses the parameters and body of +meth+
+
+  def parse_method_params_and_body container, single, meth, added_container
+    token_listener meth do
+      @scanner.continue = false
+      parse_method_parameters meth
+
+      if meth.document_self or not @track_visibility then
+        container.add_method meth
+      elsif added_container then
+        container.document_self = false
+      end
+
+      # Having now read the method parameters and documentation modifiers, we
+      # now know whether we have to rename #initialize to ::new
+
+      if meth.name == "initialize" && !meth.singleton then
+        if meth.dont_rename_initialize then
+          meth.visibility = :protected
+        else
+          meth.singleton = true
+          meth.name = "new"
+          meth.visibility = :public
+        end
+      end
+
+      parse_statements container, single, meth
+    end
+  end
+
+  ##
+  # Parses a method that needs to be ignored.
+
+  def parse_method_dummy container
+    dummy = RDoc::Context.new
+    dummy.parent = container
+    dummy.store  = container.store
+    skip_method dummy
+  end
+
+  ##
+  # Parses the name of a method in +container+.
+  #
+  # Returns the method name, the container it is in (for def Foo.name) and if
+  # it is a singleton or regular method.
+
+  def parse_method_name container # :nodoc:
+    @scanner.lex_state = :EXPR_FNAME
+
+    skip_tkspace
+    name_t = get_tk
+    back_tk = skip_tkspace
+    singleton = false
+
+    case dot = get_tk
+    when TkDOT, TkCOLON2 then
+      singleton = true
+
+      name, container = parse_method_name_singleton container, name_t
+    else
+      unget_tk dot
+      back_tk.reverse_each do |token|
+        unget_tk token
+      end
+
+      name = parse_method_name_regular container, name_t
+    end
+
+    return name, container, singleton
+  end
+
+  ##
+  # For the given +container+ and initial name token +name_t+ the method name
+  # is parsed from the token stream for a regular method.
+
+  def parse_method_name_regular container, name_t # :nodoc:
+    case name_t
+    when TkSTAR, TkAMPER then
+      name_t.text
+    else
+      unless name_t.respond_to? :name then
+        warn "expected method name token, . or ::, got #{name_t.inspect}"
+        skip_method container
+        return
+      end
+      name_t.name
+    end
+  end
+
+  ##
+  # For the given +container+ and initial name token +name_t+ the method name
+  # and the new +container+ (if necessary) are parsed from the token stream
+  # for a singleton method.
+
+  def parse_method_name_singleton container, name_t # :nodoc:
+    @scanner.lex_state = :EXPR_FNAME
+    skip_tkspace
+    name_t2 = get_tk
+
+    name =
+      case name_t
+      when TkSELF, TkMOD then
+        case name_t2
+          # NOTE: work around '[' being consumed early and not being re-tokenized
+          # as a TkAREF
+        when TkfLBRACK then
+          get_tk
+          '[]'
+        else
+          name_t2.name
+        end
+      when TkCONSTANT then
+        name = name_t2.name
+
+        container = get_method_container container, name_t
+
+        return unless container
+
+        name
+      when TkIDENTIFIER, TkIVAR, TkGVAR then
+        parse_method_dummy container
+
+        nil
+      when TkTRUE, TkFALSE, TkNIL then
+        klass_name = "#{name_t.name.capitalize}Class"
+        container = @store.find_class_named klass_name
+        container ||= @top_level.add_class RDoc::NormalClass, klass_name
+
+        name_t2.name
+      else
+        warn "unexpected method name token #{name_t.inspect}"
+        # break
+        skip_method container
+
+        nil
+      end
+
+    return name, container
+  end
+
+  ##
+  # Extracts +yield+ parameters from +method+
+
+  def parse_method_or_yield_parameters(method = nil,
+                                       modifiers = RDoc::METHOD_MODIFIERS)
+    skip_tkspace false
+    tk = get_tk
+    end_token = get_end_token tk
+    return '' unless end_token
+
+    nest = 0
+
+    loop do
+      case tk
+      when TkSEMICOLON then
+        break if nest == 0
+      when TkLBRACE, TkfLBRACE then
+        nest += 1
+      when TkRBRACE then
+        nest -= 1
+        if nest <= 0
+          # we might have a.each { |i| yield i }
+          unget_tk(tk) if nest < 0
+          break
+        end
+      when TkLPAREN, TkfLPAREN then
+        nest += 1
+      when end_token then
+        if end_token == TkRPAREN
+          nest -= 1
+          break if nest <= 0
+        else
+          break unless @scanner.continue
+        end
+      when TkRPAREN then
+        nest -= 1
+      when method && method.block_params.nil? && TkCOMMENT then
+        unget_tk tk
+        read_documentation_modifiers method, modifiers
+        @read.pop
+      when TkCOMMENT then
+        @read.pop
+      when nil then
+        break
+      end
+      tk = get_tk
+    end
+
+    get_tkread_clean(/\s+/, ' ')
+  end
+
+  ##
+  # Capture the method's parameters. Along the way, look for a comment
+  # containing:
+  #
+  #    # yields: ....
+  #
+  # and add this as the block_params for the method
+
+  def parse_method_parameters method
+    res = parse_method_or_yield_parameters method
+
+    res = "(#{res})" unless res =~ /\A\(/
+    method.params = res unless method.params
+
+    return if  method.block_params
+
+    skip_tkspace false
+    read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
+  end
+
+  ##
+  # Parses an RDoc::NormalModule in +container+ with +comment+
+
+  def parse_module container, single, tk, comment
+    container, name_t, = get_class_or_module container
+
+    name = name_t.name
+
+    mod = container.add_module RDoc::NormalModule, name
+    mod.ignore unless container.document_children
+    record_location mod
+
+    read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+    mod.add_comment comment, @top_level
+    parse_statements mod
+
+    @stats.add_module mod
+  end
+
+  ##
+  # Parses an RDoc::Require in +context+ containing +comment+
+
+  def parse_require(context, comment)
+    skip_tkspace_comment
+    tk = get_tk
+
+    if TkLPAREN === tk then
+      skip_tkspace_comment
+      tk = get_tk
+    end
+
+    name = tk.text if TkSTRING === tk
+
+    if name then
+      @top_level.add_require RDoc::Require.new(name, comment)
+    else
+      unget_tk tk
+    end
+  end
+
+  ##
+  # Parses a rescue
+
+  def parse_rescue
+    skip_tkspace false
+
+    while tk = get_tk
+      case tk
+      when TkNL, TkSEMICOLON then
+        break
+      when TkCOMMA then
+        skip_tkspace false
+
+        get_tk if TkNL === peek_tk
+      end
+
+      skip_tkspace false
+    end
+  end
+
+  ##
+  # The core of the Ruby parser.
+
+  def parse_statements(container, single = NORMAL, current_method = nil,
+                       comment = new_comment(''))
+    raise 'no' unless RDoc::Comment === comment
+    comment.force_encoding @encoding if @encoding
+
+    nest = 1
+    save_visibility = container.visibility
+
+    non_comment_seen = true
+
+    while tk = get_tk do
+      keep_comment = false
+      try_parse_comment = false
+
+      non_comment_seen = true unless TkCOMMENT === tk
+
+      case tk
+      when TkNL then
+        skip_tkspace
+        tk = get_tk
+
+        if TkCOMMENT === tk then
+          if non_comment_seen then
+            # Look for RDoc in a comment about to be thrown away
+            non_comment_seen = parse_comment container, tk, comment unless
+              comment.empty?
+
+            comment = ''
+            comment.force_encoding @encoding if @encoding
+          end
+
+          while TkCOMMENT === tk do
+            comment << tk.text << "\n"
+
+            tk = get_tk
+
+            if TkNL === tk then
+              skip_tkspace false # leading spaces
+              tk = get_tk
+            end
+          end
+
+          comment = new_comment comment
+
+          unless comment.empty? then
+            look_for_directives_in container, comment
+
+            if container.done_documenting then
+              throw :eof if RDoc::TopLevel === container
+              container.ongoing_visibility = save_visibility
+            end
+          end
+
+          keep_comment = true
+        else
+          non_comment_seen = true
+        end
+
+        unget_tk tk
+        keep_comment = true
+
+      when TkCLASS then
+        parse_class container, single, tk, comment
+
+      when TkMODULE then
+        parse_module container, single, tk, comment
+
+      when TkDEF then
+        parse_method container, single, tk, comment
+
+      when TkCONSTANT then
+        unless parse_constant container, tk, comment, current_method then
+          try_parse_comment = true
+        end
+
+      when TkALIAS then
+        parse_alias container, single, tk, comment unless current_method
+
+      when TkYIELD then
+        if current_method.nil? then
+          warn "Warning: yield outside of method" if container.document_self
+        else
+          parse_yield container, single, tk, current_method
+        end
+
+      # Until and While can have a 'do', which shouldn't increase the nesting.
+      # We can't solve the general case, but we can handle most occurrences by
+      # ignoring a do at the end of a line.
+
+      when TkUNTIL, TkWHILE then
+        nest += 1
+        skip_optional_do_after_expression
+
+      # 'for' is trickier
+      when TkFOR then
+        nest += 1
+        skip_for_variable
+        skip_optional_do_after_expression
+
+      when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then
+        nest += 1
+
+      when TkSUPER then
+        current_method.calls_super = true if current_method
+
+      when TkRESCUE then
+        parse_rescue
+
+      when TkIDENTIFIER then
+        if nest == 1 and current_method.nil? then
+          keep_comment = parse_identifier container, single, tk, comment
+        end
+
+        case tk.name
+        when "require" then
+          parse_require container, comment
+        when "include" then
+          parse_extend_or_include RDoc::Include, container, comment
+        when "extend" then
+          parse_extend_or_include RDoc::Extend, container, comment
+        end
+
+      when TkEND then
+        nest -= 1
+        if nest == 0 then
+          read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
+          container.ongoing_visibility = save_visibility
+
+          parse_comment container, tk, comment unless comment.empty?
+
+          return
+        end
+      else
+        try_parse_comment = nest == 1
+      end
+
+      if try_parse_comment then
+        non_comment_seen = parse_comment container, tk, comment unless
+          comment.empty?
+
+        keep_comment = false
+      end
+
+      unless keep_comment then
+        comment = new_comment ''
+        comment.force_encoding @encoding if @encoding
+        container.params = nil
+        container.block_params = nil
+      end
+
+      consume_trailing_spaces
+    end
+
+    container.params = nil
+    container.block_params = nil
+  end
+
+  ##
+  # Parse up to +no+ symbol arguments
+
+  def parse_symbol_arg(no = nil)
+    skip_tkspace_comment
+
+    case tk = get_tk
+    when TkLPAREN
+      parse_symbol_arg_paren no
+    else
+      parse_symbol_arg_space no, tk
+    end
+  end
+
+  ##
+  # Parses up to +no+ symbol arguments surrounded by () and places them in
+  # +args+.
+
+  def parse_symbol_arg_paren no # :nodoc:
+    args = []
+
+    loop do
+      skip_tkspace_comment
+      if tk1 = parse_symbol_in_arg
+        args.push tk1
+        break if no and args.size >= no
+      end
+
+      skip_tkspace_comment
+      case tk2 = get_tk
+      when TkRPAREN
+        break
+      when TkCOMMA
+      else
+        warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
+        break
+      end
+    end
+
+    args
+  end
+
+  ##
+  # Parses up to +no+ symbol arguments separated by spaces and places them in
+  # +args+.
+
+  def parse_symbol_arg_space no, tk # :nodoc:
+    args = []
+
+    unget_tk tk
+    if tk = parse_symbol_in_arg
+      args.push tk
+      return args if no and args.size >= no
+    end
+
+    loop do
+      skip_tkspace false
+
+      tk1 = get_tk
+      unless TkCOMMA === tk1 then
+        unget_tk tk1
+        break
+      end
+
+      skip_tkspace_comment
+      if tk = parse_symbol_in_arg
+        args.push tk
+        break if no and args.size >= no
+      end
+    end
+
+    args
+  end
+
+  ##
+  # Returns symbol text from the next token
+
+  def parse_symbol_in_arg
+    case tk = get_tk
+    when TkSYMBOL
+      tk.text.sub(/^:/, '')
+    when TkSTRING
+      eval @read[-1]
+    when TkDSTRING, TkIDENTIFIER then
+      nil # ignore
+    else
+      warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
+      nil
+    end
+  end
+
+  ##
+  # Parses statements in the top-level +container+
+
+  def parse_top_level_statements container
+    comment = collect_first_comment
+
+    look_for_directives_in container, comment
+
+    throw :eof if container.done_documenting
+
+    @markup = comment.format
+
+    # HACK move if to RDoc::Context#comment=
+    container.comment = comment if container.document_self unless comment.empty?
+
+    parse_statements container, NORMAL, nil, comment
+  end
+
+  ##
+  # Determines the visibility in +container+ from +tk+
+
+  def parse_visibility(container, single, tk)
+    vis_type, vis, singleton = get_visibility_information tk, single
+
+    skip_tkspace_comment false
+
+    case peek_tk
+      # Ryan Davis suggested the extension to ignore modifiers, because he
+      # often writes
+      #
+      #   protected unless $TESTING
+      #
+    when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then
+      container.ongoing_visibility = vis
+    else
+      update_visibility container, vis_type, vis, singleton
+    end
+  end
+
+  ##
+  # Determines the block parameter for +context+
+
+  def parse_yield(context, single, tk, method)
+    return if method.block_params
+
+    get_tkread
+    @scanner.continue = false
+    method.block_params = parse_method_or_yield_parameters
+  end
+
+  ##
+  # Directives are modifier comments that can appear after class, module, or
+  # method names. For example:
+  #
+  #   def fred # :yields: a, b
+  #
+  # or:
+  #
+  #   class MyClass # :nodoc:
+  #
+  # We return the directive name and any parameters as a two element array if
+  # the name is in +allowed+.  A directive can be found anywhere up to the end
+  # of the current line.
+
+  def read_directive allowed
+    tokens = []
+
+    while tk = get_tk do
+      tokens << tk
+
+      case tk
+      when TkNL, TkDEF then
+        return
+      when TkCOMMENT then
+        return unless tk.text =~ /\s*:?([\w-]+):\s*(.*)/
+
+        directive = $1.downcase
+
+        return [directive, $2] if allowed.include? directive
+
+        return
+      end
+    end
+  ensure
+    unless tokens.length == 1 and TkCOMMENT === tokens.first then
+      tokens.reverse_each do |token|
+        unget_tk token
+      end
+    end
+  end
+
+  ##
+  # Handles directives following the definition for +context+ (any
+  # RDoc::CodeObject) if the directives are +allowed+ at this point.
+  #
+  # See also RDoc::Markup::PreProcess#handle_directive
+
+  def read_documentation_modifiers context, allowed
+    directive, value = read_directive allowed
+
+    return unless directive
+
+    @preprocess.handle_directive '', directive, value, context do |dir, param|
+      if %w[notnew not_new not-new].include? dir then
+        context.dont_rename_initialize = true
+
+        true
+      end
+    end
+  end
+
+  ##
+  # Records the location of this +container+ in the file for this parser and
+  # adds it to the list of classes and modules in the file.
+
+  def record_location container # :nodoc:
+    case container
+    when RDoc::ClassModule then
+      @top_level.add_to_classes_or_modules container
+    end
+
+    container.record_location @top_level
+  end
+
+  ##
+  # Removes private comments from +comment+
+  #--
+  # TODO remove
+
+  def remove_private_comments comment
+    comment.remove_private
+  end
+
+  ##
+  # Scans this Ruby file for Ruby constructs
+
+  def scan
+    reset
+
+    catch :eof do
+      begin
+        parse_top_level_statements @top_level
+
+      rescue StandardError => e
+        bytes = ''
+
+        20.times do @scanner.ungetc end
+        count = 0
+        60.times do |i|
+          count = i
+          byte = @scanner.getc
+          break unless byte
+          bytes << byte
+        end
+        count -= 20
+        count.times do @scanner.ungetc end
+
+        $stderr.puts <<-EOF
+
+#{self.class} failure around line #{@scanner.line_no} of
+#{@file_name}
+
+        EOF
+
+        unless bytes.empty? then
+          $stderr.puts
+          $stderr.puts bytes.inspect
+        end
+
+        raise e
+      end
+    end
+
+    @top_level
+  end
+
+  ##
+  # while, until, and for have an optional do
+
+  def skip_optional_do_after_expression
+    skip_tkspace false
+    tk = get_tk
+    end_token = get_end_token tk
+
+    b_nest = 0
+    nest = 0
+    @scanner.continue = false
+
+    loop do
+      case tk
+      when TkSEMICOLON then
+        break if b_nest.zero?
+      when TkLPAREN, TkfLPAREN then
+        nest += 1
+      when TkBEGIN then
+        b_nest += 1
+      when TkEND then
+        b_nest -= 1
+      when TkDO
+        break if nest.zero?
+      when end_token then
+        if end_token == TkRPAREN
+          nest -= 1
+          break if @scanner.lex_state == :EXPR_END and nest.zero?
+        else
+          break unless @scanner.continue
+        end
+      when nil then
+        break
+      end
+      tk = get_tk
+    end
+
+    skip_tkspace false
+
+    get_tk if TkDO === peek_tk
+  end
+
+  ##
+  # skip the var [in] part of a 'for' statement
+
+  def skip_for_variable
+    skip_tkspace false
+    get_tk
+    skip_tkspace false
+    tk = get_tk
+    unget_tk(tk) unless TkIN === tk
+  end
+
+  ##
+  # Skips the next method in +container+
+
+  def skip_method container
+    meth = RDoc::AnyMethod.new "", "anon"
+    parse_method_parameters meth
+    parse_statements container, false, meth
+  end
+
+  ##
+  # Skip spaces until a comment is found
+
+  def skip_tkspace_comment(skip_nl = true)
+    loop do
+      skip_tkspace skip_nl
+      return unless TkCOMMENT === peek_tk
+      get_tk
+    end
+  end
+
+  ##
+  # Updates visibility in +container+ from +vis_type+ and +vis+.
+
+  def update_visibility container, vis_type, vis, singleton # :nodoc:
+    new_methods = []
+
+    case vis_type
+    when 'module_function' then
+      args = parse_symbol_arg
+      container.set_visibility_for args, :private, false
+
+      container.methods_matching args do |m|
+        s_m = m.dup
+        record_location s_m
+        s_m.singleton = true
+        new_methods << s_m
+      end
+    when 'public_class_method', 'private_class_method' then
+      args = parse_symbol_arg
+
+      container.methods_matching args, true do |m|
+        if m.parent != container then
+          m = m.dup
+          record_location m
+          new_methods << m
+        end
+
+        m.visibility = vis
+      end
+    else
+      args = parse_symbol_arg
+      container.set_visibility_for args, vis, singleton
+    end
+
+    new_methods.each do |method|
+      case method
+      when RDoc::AnyMethod then
+        container.add_method method
+      when RDoc::Attr then
+        container.add_attribute method
+      end
+      method.visibility = vis
+    end
+  end
+
+  ##
+  # Prints +message+ to +$stderr+ unless we're being quiet
+
+  def warn message
+    @options.warn make_message message
+  end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/ruby_tools.rb b/jni/ruby/lib/rdoc/parser/ruby_tools.rb
new file mode 100644
index 0000000..654431e
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/ruby_tools.rb
@@ -0,0 +1,167 @@
+##
+# Collection of methods for writing parsers against RDoc::RubyLex and
+# RDoc::RubyToken
+
+module RDoc::Parser::RubyTools
+
+  include RDoc::RubyToken
+
+  ##
+  # Adds a token listener +obj+, but you should probably use token_listener
+
+  def add_token_listener(obj)
+    @token_listeners ||= []
+    @token_listeners << obj
+  end
+
+  ##
+  # Fetches the next token from the scanner
+
+  def get_tk
+    tk = nil
+
+    if @tokens.empty? then
+      tk = @scanner.token
+      @read.push @scanner.get_readed
+      puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+    else
+      @read.push @unget_read.shift
+      tk = @tokens.shift
+      puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
+    end
+
+    tk = nil if TkEND_OF_SCRIPT === tk
+
+    if TkSYMBEG === tk then
+      set_token_position tk.line_no, tk.char_no
+
+      case tk1 = get_tk
+      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
+        if tk1.respond_to?(:name) then
+          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
+        else
+          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
+        end
+
+        # remove the identifier we just read to replace it with a symbol
+        @token_listeners.each do |obj|
+          obj.pop_token
+        end if @token_listeners
+      else
+        tk = tk1
+      end
+    end
+
+    # inform any listeners of our shiny new token
+    @token_listeners.each do |obj|
+      obj.add_token(tk)
+    end if @token_listeners
+
+    tk
+  end
+
+  ##
+  # Reads and returns all tokens up to one of +tokens+.  Leaves the matched
+  # token in the token list.
+
+  def get_tk_until(*tokens)
+    read = []
+
+    loop do
+      tk = get_tk
+
+      case tk
+      when *tokens then
+        unget_tk tk
+        break
+      end
+
+      read << tk
+    end
+
+    read
+  end
+
+  ##
+  # Retrieves a String representation of the read tokens
+
+  def get_tkread
+    read = @read.join("")
+    @read = []
+    read
+  end
+
+  ##
+  # Peek equivalent for get_tkread
+
+  def peek_read
+    @read.join('')
+  end
+
+  ##
+  # Peek at the next token, but don't remove it from the stream
+
+  def peek_tk
+    unget_tk(tk = get_tk)
+    tk
+  end
+
+  ##
+  # Removes the token listener +obj+
+
+  def remove_token_listener(obj)
+    @token_listeners.delete(obj)
+  end
+
+  ##
+  # Resets the tools
+
+  def reset
+    @read       = []
+    @tokens     = []
+    @unget_read = []
+    @nest = 0
+  end
+
+  ##
+  # Skips whitespace tokens including newlines if +skip_nl+ is true
+
+  def skip_tkspace(skip_nl = true) # HACK dup
+    tokens = []
+
+    while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
+      tokens.push tk
+    end
+
+    unget_tk tk
+    tokens
+  end
+
+  ##
+  # Has +obj+ listen to tokens
+
+  def token_listener(obj)
+    add_token_listener obj
+    yield
+  ensure
+    remove_token_listener obj
+  end
+
+  ##
+  # Returns +tk+ to the scanner
+
+  def unget_tk(tk)
+    @tokens.unshift tk
+    @unget_read.unshift @read.pop
+
+    # Remove this token from any listeners
+    @token_listeners.each do |obj|
+      obj.pop_token
+    end if @token_listeners
+
+    nil
+  end
+
+end
+
+
diff --git a/jni/ruby/lib/rdoc/parser/simple.rb b/jni/ruby/lib/rdoc/parser/simple.rb
new file mode 100644
index 0000000..65cfc1b
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/simple.rb
@@ -0,0 +1,61 @@
+##
+# Parse a non-source file. We basically take the whole thing as one big
+# comment.
+
+class RDoc::Parser::Simple < RDoc::Parser
+
+  include RDoc::Parser::Text
+
+  parse_files_matching(//)
+
+  attr_reader :content # :nodoc:
+
+  ##
+  # Prepare to parse a plain file
+
+  def initialize(top_level, file_name, content, options, stats)
+    super
+
+    preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
+
+    preprocess.handle @content, @top_level
+  end
+
+  ##
+  # Extract the file contents and attach them to the TopLevel as a comment
+
+  def scan
+    comment = remove_coding_comment @content
+    comment = remove_private_comment comment
+
+    comment = RDoc::Comment.new comment, @top_level
+
+    @top_level.comment = comment
+    @top_level
+  end
+
+  ##
+  # Removes the encoding magic comment from +text+
+
+  def remove_coding_comment text
+    text.sub(/\A# .*coding[=:].*$/, '')
+  end
+
+  ##
+  # Removes private comments.
+  #
+  # Unlike RDoc::Comment#remove_private this implementation only looks for two
+  # dashes at the beginning of the line.  Three or more dashes are considered
+  # to be a rule and ignored.
+
+  def remove_private_comment comment
+    # Workaround for gsub encoding for Ruby 1.9.2 and earlier
+    empty = ''
+    empty.force_encoding comment.encoding if Object.const_defined? :Encoding
+
+    comment = comment.gsub(%r%^--\n.*?^\+\+\n?%m, empty)
+    comment.sub(%r%^--\n.*%m, empty)
+  end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/text.rb b/jni/ruby/lib/rdoc/parser/text.rb
new file mode 100644
index 0000000..f973313
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/text.rb
@@ -0,0 +1,11 @@
+##
+# Indicates this parser is text and doesn't contain code constructs.
+#
+# Include this module in a RDoc::Parser subclass to make it show up as a file,
+# not as part of a class or module.
+#--
+# This is not named File to avoid overriding ::File
+
+module RDoc::Parser::Text
+end
+