summaryrefslogtreecommitdiff
path: root/jni/ruby/lib/rdoc/parser
diff options
context:
space:
mode:
Diffstat (limited to 'jni/ruby/lib/rdoc/parser')
-rw-r--r--jni/ruby/lib/rdoc/parser/c.rb1229
-rw-r--r--jni/ruby/lib/rdoc/parser/changelog.rb198
-rw-r--r--jni/ruby/lib/rdoc/parser/markdown.rb23
-rw-r--r--jni/ruby/lib/rdoc/parser/rd.rb22
-rw-r--r--jni/ruby/lib/rdoc/parser/ruby.rb2160
-rw-r--r--jni/ruby/lib/rdoc/parser/ruby_tools.rb167
-rw-r--r--jni/ruby/lib/rdoc/parser/simple.rb61
-rw-r--r--jni/ruby/lib/rdoc/parser/text.rb11
8 files changed, 3871 insertions, 0 deletions
diff --git a/jni/ruby/lib/rdoc/parser/c.rb b/jni/ruby/lib/rdoc/parser/c.rb
new file mode 100644
index 0000000..fd336f5
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/c.rb
@@ -0,0 +1,1229 @@
+require 'tsort'
+
+##
+# RDoc::Parser::C attempts to parse C extension files. It looks for
+# the standard patterns that you find in extensions: <tt>rb_define_class,
+# rb_define_method</tt> and so on. It tries to find the corresponding
+# C source for the methods and extract comments, but if we fail
+# we don't worry too much.
+#
+# The comments associated with a Ruby method are extracted from the C
+# comment block associated with the routine that _implements_ that
+# method, that is to say the method whose name is given in the
+# <tt>rb_define_method</tt> call. For example, you might write:
+#
+# /*
+# * Returns a new array that is a one-dimensional flattening of this
+# * array (recursively). That is, for every element that is an array,
+# * extract its elements into the new array.
+# *
+# * s = [ 1, 2, 3 ] #=> [1, 2, 3]
+# * t = [ 4, 5, 6, [7, 8] ] #=> [4, 5, 6, [7, 8]]
+# * a = [ s, t, 9, 10 ] #=> [[1, 2, 3], [4, 5, 6, [7, 8]], 9, 10]
+# * a.flatten #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+# */
+# static VALUE
+# rb_ary_flatten(ary)
+# VALUE ary;
+# {
+# ary = rb_obj_dup(ary);
+# rb_ary_flatten_bang(ary);
+# return ary;
+# }
+#
+# ...
+#
+# void
+# Init_Array()
+# {
+# ...
+# rb_define_method(rb_cArray, "flatten", rb_ary_flatten, 0);
+#
+# Here RDoc will determine from the rb_define_method line that there's a
+# method called "flatten" in class Array, and will look for the implementation
+# in the method rb_ary_flatten. It will then use the comment from that
+# method in the HTML output. This method must be in the same source file
+# as the rb_define_method.
+#
+# The comment blocks may include special directives:
+#
+# [Document-class: +name+]
+# Documentation for the named class.
+#
+# [Document-module: +name+]
+# Documentation for the named module.
+#
+# [Document-const: +name+]
+# Documentation for the named +rb_define_const+.
+#
+# Constant values can be supplied on the first line of the comment like so:
+#
+# /* 300: The highest possible score in bowling */
+# rb_define_const(cFoo, "PERFECT", INT2FIX(300));
+#
+# The value can contain internal colons so long as they are escaped with a \
+#
+# [Document-global: +name+]
+# Documentation for the named +rb_define_global_const+
+#
+# [Document-variable: +name+]
+# Documentation for the named +rb_define_variable+
+#
+# [Document-method: +method_name+]
+# Documentation for the named method. Use this when the method name is
+# unambiguous.
+#
+# [Document-method: <tt>ClassName::method_name<tt>]
+# Documentation for a singleton method in the given class. Use this when
+# the method name alone is ambiguous.
+#
+# [Document-method: <tt>ClassName#method_name<tt>]
+# Documentation for a instance method in the given class. Use this when the
+# method name alone is ambiguous.
+#
+# [Document-attr: +name+]
+# Documentation for the named attribute.
+#
+# [call-seq: <i>text up to an empty line</i>]
+# Because C source doesn't give descriptive names to Ruby-level parameters,
+# you need to document the calling sequence explicitly
+#
+# In addition, RDoc assumes by default that the C method implementing a
+# Ruby function is in the same source file as the rb_define_method call.
+# If this isn't the case, add the comment:
+#
+# rb_define_method(....); // in filename
+#
+# As an example, we might have an extension that defines multiple classes
+# in its Init_xxx method. We could document them using
+#
+# /*
+# * Document-class: MyClass
+# *
+# * Encapsulate the writing and reading of the configuration
+# * file. ...
+# */
+#
+# /*
+# * Document-method: read_value
+# *
+# * call-seq:
+# * cfg.read_value(key) -> value
+# * cfg.read_value(key} { |key| } -> value
+# *
+# * Return the value corresponding to +key+ from the configuration.
+# * In the second form, if the key isn't found, invoke the
+# * block and return its value.
+# */
+
+class RDoc::Parser::C < RDoc::Parser
+
+ parse_files_matching(/\.(?:([CcHh])\1?|c([+xp])\2|y)\z/)
+
+ include RDoc::Text
+
+ ##
+ # Maps C variable names to names of Ruby classes or modules
+
+ attr_reader :classes
+
+ ##
+ # C file the parser is parsing
+
+ attr_accessor :content
+
+ ##
+ # Dependencies from a missing enclosing class to the classes in
+ # missing_dependencies that depend upon it.
+
+ attr_reader :enclosure_dependencies
+
+ ##
+ # Maps C variable names to names of Ruby classes (and singleton classes)
+
+ attr_reader :known_classes
+
+ ##
+ # Classes found while parsing the C file that were not yet registered due to
+ # a missing enclosing class. These are processed by do_missing
+
+ attr_reader :missing_dependencies
+
+ ##
+ # Maps C variable names to names of Ruby singleton classes
+
+ attr_reader :singleton_classes
+
+ ##
+ # The TopLevel items in the parsed file belong to
+
+ attr_reader :top_level
+
+ ##
+ # Prepares for parsing a C file. See RDoc::Parser#initialize for details on
+ # the arguments.
+
+ def initialize top_level, file_name, content, options, stats
+ super
+
+ @known_classes = RDoc::KNOWN_CLASSES.dup
+ @content = handle_tab_width handle_ifdefs_in @content
+ @file_dir = File.dirname @file_name
+
+ @classes = load_variable_map :c_class_variables
+ @singleton_classes = load_variable_map :c_singleton_class_variables
+
+ # class_variable => { function => [method, ...] }
+ @methods = Hash.new { |h, f| h[f] = Hash.new { |i, m| i[m] = [] } }
+
+ # missing variable => [handle_class_module arguments]
+ @missing_dependencies = {}
+
+ # missing enclosure variable => [dependent handle_class_module arguments]
+ @enclosure_dependencies = Hash.new { |h, k| h[k] = [] }
+ @enclosure_dependencies.instance_variable_set :@missing_dependencies,
+ @missing_dependencies
+
+ @enclosure_dependencies.extend TSort
+
+ def @enclosure_dependencies.tsort_each_node &block
+ each_key(&block)
+ rescue TSort::Cyclic => e
+ cycle_vars = e.message.scan(/"(.*?)"/).flatten
+
+ cycle = cycle_vars.sort.map do |var_name|
+ delete var_name
+
+ var_name, type, mod_name, = @missing_dependencies[var_name]
+
+ "#{type} #{mod_name} (#{var_name})"
+ end.join ', '
+
+ warn "Unable to create #{cycle} due to a cyclic class or module creation"
+
+ retry
+ end
+
+ def @enclosure_dependencies.tsort_each_child node, &block
+ fetch(node, []).each(&block)
+ end
+ end
+
+ ##
+ # Removes duplicate call-seq entries for methods using the same
+ # implementation.
+
+ def deduplicate_call_seq
+ @methods.each do |var_name, functions|
+ class_name = @known_classes[var_name]
+ class_obj = find_class var_name, class_name
+
+ functions.each_value do |method_names|
+ next if method_names.length == 1
+
+ method_names.each do |method_name|
+ deduplicate_method_name class_obj, method_name
+ end
+ end
+ end
+ end
+
+ ##
+ # If two ruby methods share a C implementation (and comment) this
+ # deduplicates the examples in the call_seq for the method to reduce
+ # confusion in the output.
+
+ def deduplicate_method_name class_obj, method_name # :nodoc:
+ return unless
+ method = class_obj.method_list.find { |m| m.name == method_name }
+ return unless call_seq = method.call_seq
+
+ method_name = method_name[0, 1] if method_name =~ /\A\[/
+
+ entries = call_seq.split "\n"
+
+ matching = entries.select do |entry|
+ entry =~ /^\w*\.?#{Regexp.escape method_name}/ or
+ entry =~ /\s#{Regexp.escape method_name}\s/
+ end
+
+ method.call_seq = matching.join "\n"
+ end
+
+ ##
+ # Scans #content for rb_define_alias
+
+ def do_aliases
+ @content.scan(/rb_define_alias\s*\(
+ \s*(\w+),
+ \s*"(.+?)",
+ \s*"(.+?)"
+ \s*\)/xm) do |var_name, new_name, old_name|
+ class_name = @known_classes[var_name]
+
+ unless class_name then
+ @options.warn "Enclosing class or module %p for alias %s %s is not known" % [
+ var_name, new_name, old_name]
+ next
+ end
+
+ class_obj = find_class var_name, class_name
+
+ al = RDoc::Alias.new '', old_name, new_name, ''
+ al.singleton = @singleton_classes.key? var_name
+
+ comment = find_alias_comment var_name, new_name, old_name
+
+ comment.normalize
+
+ al.comment = comment
+
+ al.record_location @top_level
+
+ class_obj.add_alias al
+ @stats.add_alias al
+ end
+ end
+
+ ##
+ # Scans #content for rb_attr and rb_define_attr
+
+ def do_attrs
+ @content.scan(/rb_attr\s*\(
+ \s*(\w+),
+ \s*([\w"()]+),
+ \s*([01]),
+ \s*([01]),
+ \s*\w+\);/xm) do |var_name, attr_name, read, write|
+ handle_attr var_name, attr_name, read, write
+ end
+
+ @content.scan(%r%rb_define_attr\(
+ \s*([\w\.]+),
+ \s*"([^"]+)",
+ \s*(\d+),
+ \s*(\d+)\s*\);
+ %xm) do |var_name, attr_name, read, write|
+ handle_attr var_name, attr_name, read, write
+ end
+ end
+
+ ##
+ # Scans #content for boot_defclass
+
+ def do_boot_defclass
+ @content.scan(/(\w+)\s*=\s*boot_defclass\s*\(\s*"(\w+?)",\s*(\w+?)\s*\)/) do
+ |var_name, class_name, parent|
+ parent = nil if parent == "0"
+ handle_class_module(var_name, :class, class_name, parent, nil)
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_class, boot_defclass, rb_define_class_under
+ # and rb_singleton_class
+
+ def do_classes
+ do_boot_defclass
+ do_define_class
+ do_define_class_under
+ do_singleton_class
+ do_struct_define_without_accessor
+ end
+
+ ##
+ # Scans #content for rb_define_variable, rb_define_readonly_variable,
+ # rb_define_const and rb_define_global_const
+
+ def do_constants
+ @content.scan(%r%\Wrb_define_
+ ( variable |
+ readonly_variable |
+ const |
+ global_const )
+ \s*\(
+ (?:\s*(\w+),)?
+ \s*"(\w+)",
+ \s*(.*?)\s*\)\s*;
+ %xm) do |type, var_name, const_name, definition|
+ var_name = "rb_cObject" if !var_name or var_name == "rb_mKernel"
+ handle_constants type, var_name, const_name, definition
+ end
+
+ @content.scan(%r%
+ \Wrb_curses_define_const
+ \s*\(
+ \s*
+ (\w+)
+ \s*
+ \)
+ \s*;%xm) do |consts|
+ const = consts.first
+
+ handle_constants 'const', 'mCurses', const, "UINT2NUM(#{const})"
+ end
+
+ @content.scan(%r%
+ \Wrb_file_const
+ \s*\(
+ \s*
+ "([^"]+)",
+ \s*
+ (.*?)
+ \s*
+ \)
+ \s*;%xm) do |name, value|
+ handle_constants 'const', 'rb_mFConst', name, value
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_class
+
+ def do_define_class
+ # The '.' lets us handle SWIG-generated files
+ @content.scan(/([\w\.]+)\s* = \s*rb_define_class\s*
+ \(
+ \s*"(\w+)",
+ \s*(\w+)\s*
+ \)/mx) do |var_name, class_name, parent|
+ handle_class_module(var_name, :class, class_name, parent, nil)
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_class_under
+
+ def do_define_class_under
+ @content.scan(/([\w\.]+)\s* = # var_name
+ \s*rb_define_class_under\s*
+ \(
+ \s* (\w+), # under
+ \s* "(\w+)", # class_name
+ \s*
+ (?:
+ ([\w\*\s\(\)\.\->]+) | # parent_name
+ rb_path2class\("([\w:]+)"\) # path
+ )
+ \s*
+ \)
+ /mx) do |var_name, under, class_name, parent_name, path|
+ parent = path || parent_name
+
+ handle_class_module var_name, :class, class_name, parent, under
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_module
+
+ def do_define_module
+ @content.scan(/(\w+)\s* = \s*rb_define_module\s*\(\s*"(\w+)"\s*\)/mx) do
+ |var_name, class_name|
+ handle_class_module(var_name, :module, class_name, nil, nil)
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_module_under
+
+ def do_define_module_under
+ @content.scan(/(\w+)\s* = \s*rb_define_module_under\s*
+ \(
+ \s*(\w+),
+ \s*"(\w+)"
+ \s*\)/mx) do |var_name, in_module, class_name|
+ handle_class_module(var_name, :module, class_name, nil, in_module)
+ end
+ end
+
+ ##
+ # Scans #content for rb_include_module
+
+ def do_includes
+ @content.scan(/rb_include_module\s*\(\s*(\w+?),\s*(\w+?)\s*\)/) do |c,m|
+ next unless cls = @classes[c]
+ m = @known_classes[m] || m
+
+ comment = RDoc::Comment.new '', @top_level
+ incl = cls.add_include RDoc::Include.new(m, comment)
+ incl.record_location @top_level
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_method, rb_define_singleton_method,
+ # rb_define_module_function, rb_define_private_method,
+ # rb_define_global_function and define_filetest_function
+
+ def do_methods
+ @content.scan(%r%rb_define_
+ (
+ singleton_method |
+ method |
+ module_function |
+ private_method
+ )
+ \s*\(\s*([\w\.]+),
+ \s*"([^"]+)",
+ \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\(|\(METHOD\))?(\w+)\)?,
+ \s*(-?\w+)\s*\)
+ (?:;\s*/[*/]\s+in\s+(\w+?\.(?:cpp|c|y)))?
+ %xm) do |type, var_name, meth_name, function, param_count, source_file|
+
+ # Ignore top-object and weird struct.c dynamic stuff
+ next if var_name == "ruby_top_self"
+ next if var_name == "nstr"
+
+ var_name = "rb_cObject" if var_name == "rb_mKernel"
+ handle_method(type, var_name, meth_name, function, param_count,
+ source_file)
+ end
+
+ @content.scan(%r%rb_define_global_function\s*\(
+ \s*"([^"]+)",
+ \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
+ \s*(-?\w+)\s*\)
+ (?:;\s*/[*/]\s+in\s+(\w+?\.[cy]))?
+ %xm) do |meth_name, function, param_count, source_file|
+ handle_method("method", "rb_mKernel", meth_name, function, param_count,
+ source_file)
+ end
+
+ @content.scan(/define_filetest_function\s*\(
+ \s*"([^"]+)",
+ \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
+ \s*(-?\w+)\s*\)/xm) do |meth_name, function, param_count|
+
+ handle_method("method", "rb_mFileTest", meth_name, function, param_count)
+ handle_method("singleton_method", "rb_cFile", meth_name, function,
+ param_count)
+ end
+ end
+
+ ##
+ # Creates classes and module that were missing were defined due to the file
+ # order being different than the declaration order.
+
+ def do_missing
+ return if @missing_dependencies.empty?
+
+ @enclosure_dependencies.tsort.each do |in_module|
+ arguments = @missing_dependencies.delete in_module
+
+ next unless arguments # dependency on existing class
+
+ handle_class_module(*arguments)
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_module and rb_define_module_under
+
+ def do_modules
+ do_define_module
+ do_define_module_under
+ end
+
+ ##
+ # Scans #content for rb_singleton_class
+
+ def do_singleton_class
+ @content.scan(/([\w\.]+)\s* = \s*rb_singleton_class\s*
+ \(
+ \s*(\w+)
+ \s*\)/mx) do |sclass_var, class_var|
+ handle_singleton sclass_var, class_var
+ end
+ end
+
+ ##
+ # Scans #content for struct_define_without_accessor
+
+ def do_struct_define_without_accessor
+ @content.scan(/([\w\.]+)\s* = \s*rb_struct_define_without_accessor\s*
+ \(
+ \s*"(\w+)", # Class name
+ \s*(\w+), # Parent class
+ \s*\w+, # Allocation function
+ (\s*"\w+",)* # Attributes
+ \s*NULL
+ \)/mx) do |var_name, class_name, parent|
+ handle_class_module(var_name, :class, class_name, parent, nil)
+ end
+ end
+
+ ##
+ # Finds the comment for an alias on +class_name+ from +new_name+ to
+ # +old_name+
+
+ def find_alias_comment class_name, new_name, old_name
+ content =~ %r%((?>/\*.*?\*/\s+))
+ rb_define_alias\(\s*#{Regexp.escape class_name}\s*,
+ \s*"#{Regexp.escape new_name}"\s*,
+ \s*"#{Regexp.escape old_name}"\s*\);%xm
+
+ RDoc::Comment.new($1 || '', @top_level)
+ end
+
+ ##
+ # Finds a comment for rb_define_attr, rb_attr or Document-attr.
+ #
+ # +var_name+ is the C class variable the attribute is defined on.
+ # +attr_name+ is the attribute's name.
+ #
+ # +read+ and +write+ are the read/write flags ('1' or '0'). Either both or
+ # neither must be provided.
+
+ def find_attr_comment var_name, attr_name, read = nil, write = nil
+ attr_name = Regexp.escape attr_name
+
+ rw = if read and write then
+ /\s*#{read}\s*,\s*#{write}\s*/xm
+ else
+ /.*?/m
+ end
+
+ comment = if @content =~ %r%((?>/\*.*?\*/\s+))
+ rb_define_attr\((?:\s*#{var_name},)?\s*
+ "#{attr_name}"\s*,
+ #{rw}\)\s*;%xm then
+ $1
+ elsif @content =~ %r%((?>/\*.*?\*/\s+))
+ rb_attr\(\s*#{var_name}\s*,
+ \s*#{attr_name}\s*,
+ #{rw},.*?\)\s*;%xm then
+ $1
+ elsif @content =~ %r%(/\*.*?(?:\s*\*\s*)?)
+ Document-attr:\s#{attr_name}\s*?\n
+ ((?>(.|\n)*?\*/))%x then
+ "#{$1}\n#{$2}"
+ else
+ ''
+ end
+
+ RDoc::Comment.new comment, @top_level
+ end
+
+ ##
+ # Find the C code corresponding to a Ruby method
+
+ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
+ case file_content
+ when %r%((?>/\*.*?\*/\s*)?)
+ ((?:(?:\w+)\s+)?
+ (?:intern\s+)?VALUE\s+#{meth_name}
+ \s*(\([^)]*\))([^;]|$))%xm then
+ comment = RDoc::Comment.new $1, @top_level
+ body = $2
+ offset, = $~.offset(2)
+
+ comment.remove_private if comment
+
+ # try to find the whole body
+ body = $& if /#{Regexp.escape body}[^(]*?\{.*?^\}/m =~ file_content
+
+ # The comment block may have been overridden with a 'Document-method'
+ # block. This happens in the interpreter when multiple methods are
+ # vectored through to the same C method but those methods are logically
+ # distinct (for example Kernel.hash and Kernel.object_id share the same
+ # implementation
+
+ override_comment = find_override_comment class_name, meth_obj
+ comment = override_comment if override_comment
+
+ comment.normalize
+ find_modifiers comment, meth_obj if comment
+
+ #meth_obj.params = params
+ meth_obj.start_collecting_tokens
+ tk = RDoc::RubyToken::Token.new nil, 1, 1
+ tk.set_text body
+ meth_obj.add_token tk
+ meth_obj.comment = comment
+ meth_obj.offset = offset
+ meth_obj.line = file_content[0, offset].count("\n") + 1
+
+ body
+ when %r%((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+#{meth_name}\s+(\w+))%m then
+ comment = RDoc::Comment.new $1, @top_level
+ body = $2
+ offset = $~.offset(2).first
+
+ find_body class_name, $3, meth_obj, file_content, true
+
+ comment.normalize
+ find_modifiers comment, meth_obj
+
+ meth_obj.start_collecting_tokens
+ tk = RDoc::RubyToken::Token.new nil, 1, 1
+ tk.set_text body
+ meth_obj.add_token tk
+ meth_obj.comment = comment
+ meth_obj.offset = offset
+ meth_obj.line = file_content[0, offset].count("\n") + 1
+
+ body
+ when %r%^\s*\#\s*define\s+#{meth_name}\s+(\w+)%m then
+ # with no comment we hope the aliased definition has it and use it's
+ # definition
+
+ body = find_body(class_name, $1, meth_obj, file_content, true)
+
+ return body if body
+
+ @options.warn "No definition for #{meth_name}"
+ false
+ else # No body, but might still have an override comment
+ comment = find_override_comment class_name, meth_obj
+
+ if comment then
+ comment.normalize
+ find_modifiers comment, meth_obj
+ meth_obj.comment = comment
+
+ ''
+ else
+ @options.warn "No definition for #{meth_name}"
+ false
+ end
+ end
+ end
+
+ ##
+ # Finds a RDoc::NormalClass or RDoc::NormalModule for +raw_name+
+
+ def find_class(raw_name, name)
+ unless @classes[raw_name]
+ if raw_name =~ /^rb_m/
+ container = @top_level.add_module RDoc::NormalModule, name
+ else
+ container = @top_level.add_class RDoc::NormalClass, name
+ end
+
+ container.record_location @top_level
+ @classes[raw_name] = container
+ end
+ @classes[raw_name]
+ end
+
+ ##
+ # Look for class or module documentation above Init_+class_name+(void),
+ # in a Document-class +class_name+ (or module) comment or above an
+ # rb_define_class (or module). If a comment is supplied above a matching
+ # Init_ and a rb_define_class the Init_ comment is used.
+ #
+ # /*
+ # * This is a comment for Foo
+ # */
+ # Init_Foo(void) {
+ # VALUE cFoo = rb_define_class("Foo", rb_cObject);
+ # }
+ #
+ # /*
+ # * Document-class: Foo
+ # * This is a comment for Foo
+ # */
+ # Init_foo(void) {
+ # VALUE cFoo = rb_define_class("Foo", rb_cObject);
+ # }
+ #
+ # /*
+ # * This is a comment for Foo
+ # */
+ # VALUE cFoo = rb_define_class("Foo", rb_cObject);
+
+ def find_class_comment class_name, class_mod
+ comment = nil
+
+ if @content =~ %r%
+ ((?>/\*.*?\*/\s+))
+ (static\s+)?
+ void\s+
+ Init_#{class_name}\s*(?:_\(\s*)?\(\s*(?:void\s*)?\)%xmi then
+ comment = $1.sub(%r%Document-(?:class|module):\s+#{class_name}%, '')
+ elsif @content =~ %r%Document-(?:class|module):\s+#{class_name}\s*?
+ (?:<\s+[:,\w]+)?\n((?>.*?\*/))%xm then
+ comment = "/*\n#{$1}"
+ elsif @content =~ %r%((?>/\*.*?\*/\s+))
+ ([\w\.\s]+\s* = \s+)?rb_define_(class|module)[\t (]*?"(#{class_name})"%xm then
+ comment = $1
+ elsif @content =~ %r%((?>/\*.*?\*/\s+))
+ ([\w\. \t]+ = \s+)?rb_define_(class|module)_under[\t\w, (]*?"(#{class_name.split('::').last})"%xm then
+ comment = $1
+ else
+ comment = ''
+ end
+
+ comment = RDoc::Comment.new comment, @top_level
+ comment.normalize
+
+ look_for_directives_in class_mod, comment
+
+ class_mod.add_comment comment, @top_level
+ end
+
+ ##
+ # Finds a comment matching +type+ and +const_name+ either above the
+ # comment or in the matching Document- section.
+
+ def find_const_comment(type, const_name, class_name = nil)
+ comment = if @content =~ %r%((?>^\s*/\*.*?\*/\s+))
+ rb_define_#{type}\((?:\s*(\w+),)?\s*
+ "#{const_name}"\s*,
+ .*?\)\s*;%xmi then
+ $1
+ elsif class_name and
+ @content =~ %r%Document-(?:const|global|variable):\s
+ #{class_name}::#{const_name}
+ \s*?\n((?>.*?\*/))%xm then
+ "/*\n#{$1}"
+ elsif @content =~ %r%Document-(?:const|global|variable):
+ \s#{const_name}
+ \s*?\n((?>.*?\*/))%xm then
+ "/*\n#{$1}"
+ else
+ ''
+ end
+
+ RDoc::Comment.new comment, @top_level
+ end
+
+ ##
+ # Handles modifiers in +comment+ and updates +meth_obj+ as appropriate.
+
+ def find_modifiers comment, meth_obj
+ comment.normalize
+ comment.extract_call_seq meth_obj
+
+ look_for_directives_in meth_obj, comment
+ end
+
+ ##
+ # Finds a <tt>Document-method</tt> override for +meth_obj+ on +class_name+
+
+ def find_override_comment class_name, meth_obj
+ name = Regexp.escape meth_obj.name
+ prefix = Regexp.escape meth_obj.name_prefix
+
+ comment = if @content =~ %r%Document-method:
+ \s+#{class_name}#{prefix}#{name}
+ \s*?\n((?>.*?\*/))%xm then
+ "/*#{$1}"
+ elsif @content =~ %r%Document-method:
+ \s#{name}\s*?\n((?>.*?\*/))%xm then
+ "/*#{$1}"
+ end
+
+ return unless comment
+
+ RDoc::Comment.new comment, @top_level
+ end
+
+ ##
+ # Creates a new RDoc::Attr +attr_name+ on class +var_name+ that is either
+ # +read+, +write+ or both
+
+ def handle_attr(var_name, attr_name, read, write)
+ rw = ''
+ rw << 'R' if '1' == read
+ rw << 'W' if '1' == write
+
+ class_name = @known_classes[var_name]
+
+ return unless class_name
+
+ class_obj = find_class var_name, class_name
+
+ return unless class_obj
+
+ comment = find_attr_comment var_name, attr_name
+ comment.normalize
+
+ name = attr_name.gsub(/rb_intern\("([^"]+)"\)/, '\1')
+
+ attr = RDoc::Attr.new '', name, rw, comment
+
+ attr.record_location @top_level
+ class_obj.add_attribute attr
+ @stats.add_attribute attr
+ end
+
+ ##
+ # Creates a new RDoc::NormalClass or RDoc::NormalModule based on +type+
+ # named +class_name+ in +parent+ which was assigned to the C +var_name+.
+
+ def handle_class_module(var_name, type, class_name, parent, in_module)
+ parent_name = @known_classes[parent] || parent
+
+ if in_module then
+ enclosure = @classes[in_module] || @store.find_c_enclosure(in_module)
+
+ if enclosure.nil? and enclosure = @known_classes[in_module] then
+ enc_type = /^rb_m/ =~ in_module ? :module : :class
+ handle_class_module in_module, enc_type, enclosure, nil, nil
+ enclosure = @classes[in_module]
+ end
+
+ unless enclosure then
+ @enclosure_dependencies[in_module] << var_name
+ @missing_dependencies[var_name] =
+ [var_name, type, class_name, parent, in_module]
+
+ return
+ end
+ else
+ enclosure = @top_level
+ end
+
+ if type == :class then
+ full_name = if RDoc::ClassModule === enclosure then
+ enclosure.full_name + "::#{class_name}"
+ else
+ class_name
+ end
+
+ if @content =~ %r%Document-class:\s+#{full_name}\s*<\s+([:,\w]+)% then
+ parent_name = $1
+ end
+
+ cm = enclosure.add_class RDoc::NormalClass, class_name, parent_name
+ else
+ cm = enclosure.add_module RDoc::NormalModule, class_name
+ end
+
+ cm.record_location enclosure.top_level
+
+ find_class_comment cm.full_name, cm
+
+ case cm
+ when RDoc::NormalClass
+ @stats.add_class cm
+ when RDoc::NormalModule
+ @stats.add_module cm
+ end
+
+ @classes[var_name] = cm
+ @known_classes[var_name] = cm.full_name
+ @store.add_c_enclosure var_name, cm
+ end
+
+ ##
+ # Adds constants. By providing some_value: at the start of the comment you
+ # can override the C value of the comment to give a friendly definition.
+ #
+ # /* 300: The perfect score in bowling */
+ # rb_define_const(cFoo, "PERFECT", INT2FIX(300);
+ #
+ # Will override <tt>INT2FIX(300)</tt> with the value +300+ in the output
+ # RDoc. Values may include quotes and escaped colons (\:).
+
+ def handle_constants(type, var_name, const_name, definition)
+ class_name = @known_classes[var_name]
+
+ return unless class_name
+
+ class_obj = find_class var_name, class_name
+
+ unless class_obj then
+ @options.warn 'Enclosing class or module %p is not known' % [const_name]
+ return
+ end
+
+ comment = find_const_comment type, const_name, class_name
+ comment.normalize
+
+ # In the case of rb_define_const, the definition and comment are in
+ # "/* definition: comment */" form. The literal ':' and '\' characters
+ # can be escaped with a backslash.
+ if type.downcase == 'const' then
+ no_match, new_definition, new_comment = comment.text.split(/(\A.*):/)
+
+ if no_match and no_match.empty? then
+ if new_definition.empty? then # Default to literal C definition
+ new_definition = definition
+ else
+ new_definition.gsub!("\:", ":")
+ new_definition.gsub!("\\", '\\')
+ end
+
+ new_definition.sub!(/\A(\s+)/, '')
+
+ new_comment = "#{$1}#{new_comment.lstrip}"
+
+ new_comment = RDoc::Comment.new new_comment, @top_level
+
+ con = RDoc::Constant.new const_name, new_definition, new_comment
+ else
+ con = RDoc::Constant.new const_name, definition, comment
+ end
+ else
+ con = RDoc::Constant.new const_name, definition, comment
+ end
+
+ con.record_location @top_level
+ @stats.add_constant con
+ class_obj.add_constant con
+ end
+
+ ##
+ # Removes #ifdefs that would otherwise confuse us
+
+ def handle_ifdefs_in(body)
+ body.gsub(/^#ifdef HAVE_PROTOTYPES.*?#else.*?\n(.*?)#endif.*?\n/m, '\1')
+ end
+
+ ##
+ # Adds an RDoc::AnyMethod +meth_name+ defined on a class or module assigned
+ # to +var_name+. +type+ is the type of method definition function used.
+ # +singleton_method+ and +module_function+ create a singleton method.
+
+ def handle_method(type, var_name, meth_name, function, param_count,
+ source_file = nil)
+ class_name = @known_classes[var_name]
+ singleton = @singleton_classes.key? var_name
+
+ @methods[var_name][function] << meth_name
+
+ return unless class_name
+
+ class_obj = find_class var_name, class_name
+
+ if class_obj then
+ if meth_name == 'initialize' then
+ meth_name = 'new'
+ singleton = true
+ type = 'method' # force public
+ end
+
+ meth_obj = RDoc::AnyMethod.new '', meth_name
+ meth_obj.c_function = function
+ meth_obj.singleton =
+ singleton || %w[singleton_method module_function].include?(type)
+
+ p_count = Integer(param_count) rescue -1
+
+ if source_file then
+ file_name = File.join @file_dir, source_file
+
+ if File.exist? file_name then
+ file_content = File.read file_name
+ else
+ @options.warn "unknown source #{source_file} for #{meth_name} in #{@file_name}"
+ end
+ else
+ file_content = @content
+ end
+
+ body = find_body class_name, function, meth_obj, file_content
+
+ if body and meth_obj.document_self then
+ meth_obj.params = if p_count < -1 then # -2 is Array
+ '(*args)'
+ elsif p_count == -1 then # argc, argv
+ rb_scan_args body
+ else
+ "(#{(1..p_count).map { |i| "p#{i}" }.join ', '})"
+ end
+
+
+ meth_obj.record_location @top_level
+ class_obj.add_method meth_obj
+ @stats.add_method meth_obj
+ meth_obj.visibility = :private if 'private_method' == type
+ end
+ end
+ end
+
+ ##
+ # Registers a singleton class +sclass_var+ as a singleton of +class_var+
+
+ def handle_singleton sclass_var, class_var
+ class_name = @known_classes[class_var]
+
+ @known_classes[sclass_var] = class_name
+ @singleton_classes[sclass_var] = class_name
+ end
+
+ ##
+ # Normalizes tabs in +body+
+
+ def handle_tab_width(body)
+ if /\t/ =~ body
+ tab_width = @options.tab_width
+ body.split(/\n/).map do |line|
+ 1 while line.gsub!(/\t+/) do
+ ' ' * (tab_width * $&.length - $`.length % tab_width)
+ end && $~
+ line
+ end.join "\n"
+ else
+ body
+ end
+ end
+
+ ##
+ # Loads the variable map with the given +name+ from the RDoc::Store, if
+ # present.
+
+ def load_variable_map map_name
+ return {} unless files = @store.cache[map_name]
+ return {} unless name_map = files[@file_name]
+
+ class_map = {}
+
+ name_map.each do |variable, name|
+ next unless mod = @store.find_class_or_module(name)
+
+ class_map[variable] = if map_name == :c_class_variables then
+ mod
+ else
+ name
+ end
+ @known_classes[variable] = name
+ end
+
+ class_map
+ end
+
+ ##
+ # Look for directives in a normal comment block:
+ #
+ # /*
+ # * :title: My Awesome Project
+ # */
+ #
+ # This method modifies the +comment+
+
+ def look_for_directives_in context, comment
+ @preprocess.handle comment, context do |directive, param|
+ case directive
+ when 'main' then
+ @options.main_page = param
+ ''
+ when 'title' then
+ @options.default_title = param if @options.respond_to? :default_title=
+ ''
+ end
+ end
+
+ comment
+ end
+
+ ##
+ # Extracts parameters from the +method_body+ and returns a method
+ # parameter string. Follows 1.9.3dev's scan-arg-spec, see README.EXT
+
+ def rb_scan_args method_body
+ method_body =~ /rb_scan_args\((.*?)\)/m
+ return '(*args)' unless $1
+
+ $1.split(/,/)[2] =~ /"(.*?)"/ # format argument
+ format = $1.split(//)
+
+ lead = opt = trail = 0
+
+ if format.first =~ /\d/ then
+ lead = $&.to_i
+ format.shift
+ if format.first =~ /\d/ then
+ opt = $&.to_i
+ format.shift
+ if format.first =~ /\d/ then
+ trail = $&.to_i
+ format.shift
+ block_arg = true
+ end
+ end
+ end
+
+ if format.first == '*' and not block_arg then
+ var = true
+ format.shift
+ if format.first =~ /\d/ then
+ trail = $&.to_i
+ format.shift
+ end
+ end
+
+ if format.first == ':' then
+ hash = true
+ format.shift
+ end
+
+ if format.first == '&' then
+ block = true
+ format.shift
+ end
+
+ # if the format string is not empty there's a bug in the C code, ignore it
+
+ args = []
+ position = 1
+
+ (1...(position + lead)).each do |index|
+ args << "p#{index}"
+ end
+
+ position += lead
+
+ (position...(position + opt)).each do |index|
+ args << "p#{index} = v#{index}"
+ end
+
+ position += opt
+
+ if var then
+ args << '*args'
+ position += 1
+ end
+
+ (position...(position + trail)).each do |index|
+ args << "p#{index}"
+ end
+
+ position += trail
+
+ if hash then
+ args << "p#{position} = {}"
+ end
+
+ args << '&block' if block
+
+ "(#{args.join ', '})"
+ end
+
+ ##
+ # Removes lines that are commented out that might otherwise get picked up
+ # when scanning for classes and methods
+
+ def remove_commented_out_lines
+ @content.gsub!(%r%//.*rb_define_%, '//')
+ end
+
+ ##
+ # Extracts the classes, modules, methods, attributes, constants and aliases
+ # from a C file and returns an RDoc::TopLevel for this file
+
+ def scan
+ remove_commented_out_lines
+
+ do_modules
+ do_classes
+ do_missing
+
+ do_constants
+ do_methods
+ do_includes
+ do_aliases
+ do_attrs
+
+ deduplicate_call_seq
+
+ @store.add_c_variables self
+
+ @top_level
+ end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/changelog.rb b/jni/ruby/lib/rdoc/parser/changelog.rb
new file mode 100644
index 0000000..a3567c1
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/changelog.rb
@@ -0,0 +1,198 @@
+require 'time'
+
+##
+# A ChangeLog file parser.
+#
+# This parser converts a ChangeLog into an RDoc::Markup::Document. When
+# viewed as HTML a ChangeLog page will have an entry for each day's entries in
+# the sidebar table of contents.
+#
+# This parser is meant to parse the MRI ChangeLog, but can be used to parse any
+# {GNU style Change
+# Log}[http://www.gnu.org/prep/standards/html_node/Style-of-Change-Logs.html].
+
+class RDoc::Parser::ChangeLog < RDoc::Parser
+
+ include RDoc::Parser::Text
+
+ parse_files_matching(/(\/|\\|\A)ChangeLog[^\/\\]*\z/)
+
+ ##
+ # Attaches the +continuation+ of the previous line to the +entry_body+.
+ #
+ # Continued function listings are joined together as a single entry.
+ # Continued descriptions are joined to make a single paragraph.
+
+ def continue_entry_body entry_body, continuation
+ return unless last = entry_body.last
+
+ if last =~ /\)\s*\z/ and continuation =~ /\A\(/ then
+ last.sub!(/\)\s*\z/, ',')
+ continuation.sub!(/\A\(/, '')
+ end
+
+ if last =~ /\s\z/ then
+ last << continuation
+ else
+ last << ' ' << continuation
+ end
+ end
+
+ ##
+ # Creates an RDoc::Markup::Document given the +groups+ of ChangeLog entries.
+
+ def create_document groups
+ doc = RDoc::Markup::Document.new
+ doc.omit_headings_below = 2
+ doc.file = @top_level
+
+ doc << RDoc::Markup::Heading.new(1, File.basename(@file_name))
+ doc << RDoc::Markup::BlankLine.new
+
+ groups.sort_by do |day,| day end.reverse_each do |day, entries|
+ doc << RDoc::Markup::Heading.new(2, day.dup)
+ doc << RDoc::Markup::BlankLine.new
+
+ doc.concat create_entries entries
+ end
+
+ doc
+ end
+
+ ##
+ # Returns a list of ChangeLog entries an RDoc::Markup nodes for the given
+ # +entries+.
+
+ def create_entries entries
+ out = []
+
+ entries.each do |entry, items|
+ out << RDoc::Markup::Heading.new(3, entry)
+ out << RDoc::Markup::BlankLine.new
+
+ out << create_items(items)
+ end
+
+ out
+ end
+
+ ##
+ # Returns an RDoc::Markup::List containing the given +items+ in the
+ # ChangeLog
+
+ def create_items items
+ list = RDoc::Markup::List.new :NOTE
+
+ items.each do |item|
+ item =~ /\A(.*?(?:\([^)]+\))?):\s*/
+
+ title = $1
+ body = $'
+
+ paragraph = RDoc::Markup::Paragraph.new body
+ list_item = RDoc::Markup::ListItem.new title, paragraph
+ list << list_item
+ end
+
+ list
+ end
+
+ ##
+ # Groups +entries+ by date.
+
+ def group_entries entries
+ entries.group_by do |title, _|
+ begin
+ Time.parse(title).strftime '%Y-%m-%d'
+ rescue NoMethodError, ArgumentError
+ time, = title.split ' ', 2
+ Time.parse(time).strftime '%Y-%m-%d'
+ end
+ end
+ end
+
+ ##
+ # Parses the entries in the ChangeLog.
+ #
+ # Returns an Array of each ChangeLog entry in order of parsing.
+ #
+ # A ChangeLog entry is an Array containing the ChangeLog title (date and
+ # committer) and an Array of ChangeLog items (file and function changed with
+ # description).
+ #
+ # An example result would be:
+ #
+ # [ 'Tue Dec 4 08:33:46 2012 Eric Hodel <drbrain@segment7.net>',
+ # [ 'README.EXT: Converted to RDoc format',
+ # 'README.EXT.ja: ditto']]
+
+ def parse_entries
+ entries = []
+ entry_name = nil
+ entry_body = []
+
+ @content.each_line do |line|
+ case line
+ when /^\s*$/ then
+ next
+ when /^\w.*/ then
+ entries << [entry_name, entry_body] if entry_name
+
+ entry_name = $&
+
+ begin
+ time = Time.parse entry_name
+ # HACK Ruby 1.8 does not raise ArgumentError for Time.parse "Other"
+ entry_name = nil unless entry_name =~ /#{time.year}/
+ rescue NoMethodError
+ # HACK Ruby 2.1.2 and earlier raises NoMethodError if time part is absent
+ entry_name.split ' ', 2
+ rescue ArgumentError
+ if /out of range/ =~ $!.message
+ Time.parse(entry_name.split(' ', 2)[0]) rescue entry_name = nil
+ else
+ entry_name = nil
+ end
+ end
+
+ entry_body = []
+ when /^(\t| {8})?\*\s*(.*)/ then # "\t* file.c (func): ..."
+ entry_body << $2
+ when /^(\t| {8})?\s*(\(.*)/ then # "\t(func): ..."
+ entry = $2
+
+ if entry_body.last =~ /:/ then
+ entry_body << entry
+ else
+ continue_entry_body entry_body, entry
+ end
+ when /^(\t| {8})?\s*(.*)/ then
+ continue_entry_body entry_body, $2
+ end
+ end
+
+ entries << [entry_name, entry_body] if entry_name
+
+ entries.reject! do |(entry,_)|
+ entry == nil
+ end
+
+ entries
+ end
+
+ ##
+ # Converts the ChangeLog into an RDoc::Markup::Document
+
+ def scan
+ entries = parse_entries
+ grouped_entries = group_entries entries
+
+ doc = create_document grouped_entries
+
+ @top_level.comment = doc
+
+ @top_level
+ end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/markdown.rb b/jni/ruby/lib/rdoc/parser/markdown.rb
new file mode 100644
index 0000000..6fd88cf
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/markdown.rb
@@ -0,0 +1,23 @@
+##
+# Parse a Markdown format file. The parsed RDoc::Markup::Document is attached
+# as a file comment.
+
+class RDoc::Parser::Markdown < RDoc::Parser
+
+ include RDoc::Parser::Text
+
+ parse_files_matching(/\.(md|markdown)(?:\.[^.]+)?$/)
+
+ ##
+ # Creates an Markdown-format TopLevel for the given file.
+
+ def scan
+ comment = RDoc::Comment.new @content, @top_level
+ comment.format = 'markdown'
+
+ @top_level.comment = comment
+ end
+
+end
+
+
diff --git a/jni/ruby/lib/rdoc/parser/rd.rb b/jni/ruby/lib/rdoc/parser/rd.rb
new file mode 100644
index 0000000..09069ae
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/rd.rb
@@ -0,0 +1,22 @@
+##
+# Parse a RD format file. The parsed RDoc::Markup::Document is attached as a
+# file comment.
+
+class RDoc::Parser::RD < RDoc::Parser
+
+ include RDoc::Parser::Text
+
+ parse_files_matching(/\.rd(?:\.[^.]+)?$/)
+
+ ##
+ # Creates an rd-format TopLevel for the given file.
+
+ def scan
+ comment = RDoc::Comment.new @content, @top_level
+ comment.format = 'rd'
+
+ @top_level.comment = comment
+ end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/ruby.rb b/jni/ruby/lib/rdoc/parser/ruby.rb
new file mode 100644
index 0000000..ce1083e
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/ruby.rb
@@ -0,0 +1,2160 @@
+##
+# This file contains stuff stolen outright from:
+#
+# rtags.rb -
+# ruby-lex.rb - ruby lexcal analyzer
+# ruby-token.rb - ruby tokens
+# by Keiju ISHITSUKA (Nippon Rational Inc.)
+#
+
+$TOKEN_DEBUG ||= nil
+
+##
+# Extracts code elements from a source file returning a TopLevel object
+# containing the constituent file elements.
+#
+# This file is based on rtags
+#
+# RubyParser understands how to document:
+# * classes
+# * modules
+# * methods
+# * constants
+# * aliases
+# * private, public, protected
+# * private_class_function, public_class_function
+# * module_function
+# * attr, attr_reader, attr_writer, attr_accessor
+# * extra accessors given on the command line
+# * metaprogrammed methods
+# * require
+# * include
+#
+# == Method Arguments
+#
+#--
+# NOTE: I don't think this works, needs tests, remove the paragraph following
+# this block when known to work
+#
+# The parser extracts the arguments from the method definition. You can
+# override this with a custom argument definition using the :args: directive:
+#
+# ##
+# # This method tries over and over until it is tired
+#
+# def go_go_go(thing_to_try, tries = 10) # :args: thing_to_try
+# puts thing_to_try
+# go_go_go thing_to_try, tries - 1
+# end
+#
+# If you have a more-complex set of overrides you can use the :call-seq:
+# directive:
+#++
+#
+# The parser extracts the arguments from the method definition. You can
+# override this with a custom argument definition using the :call-seq:
+# directive:
+#
+# ##
+# # This method can be called with a range or an offset and length
+# #
+# # :call-seq:
+# # my_method(Range)
+# # my_method(offset, length)
+#
+# def my_method(*args)
+# end
+#
+# The parser extracts +yield+ expressions from method bodies to gather the
+# yielded argument names. If your method manually calls a block instead of
+# yielding or you want to override the discovered argument names use
+# the :yields: directive:
+#
+# ##
+# # My method is awesome
+#
+# def my_method(&block) # :yields: happy, times
+# block.call 1, 2
+# end
+#
+# == Metaprogrammed Methods
+#
+# To pick up a metaprogrammed method, the parser looks for a comment starting
+# with '##' before an identifier:
+#
+# ##
+# # This is a meta-programmed method!
+#
+# add_my_method :meta_method, :arg1, :arg2
+#
+# The parser looks at the token after the identifier to determine the name, in
+# this example, :meta_method. If a name cannot be found, a warning is printed
+# and 'unknown is used.
+#
+# You can force the name of a method using the :method: directive:
+#
+# ##
+# # :method: some_method!
+#
+# By default, meta-methods are instance methods. To indicate that a method is
+# a singleton method instead use the :singleton-method: directive:
+#
+# ##
+# # :singleton-method:
+#
+# You can also use the :singleton-method: directive with a name:
+#
+# ##
+# # :singleton-method: some_method!
+#
+# You can define arguments for metaprogrammed methods via either the
+# :call-seq:, :arg: or :args: directives.
+#
+# Additionally you can mark a method as an attribute by
+# using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:. Just like
+# for :method:, the name is optional.
+#
+# ##
+# # :attr_reader: my_attr_name
+#
+# == Hidden methods and attributes
+#
+# You can provide documentation for methods that don't appear using
+# the :method:, :singleton-method: and :attr: directives:
+#
+# ##
+# # :attr_writer: ghost_writer
+# # There is an attribute here, but you can't see it!
+#
+# ##
+# # :method: ghost_method
+# # There is a method here, but you can't see it!
+#
+# ##
+# # this is a comment for a regular method
+#
+# def regular_method() end
+#
+# Note that by default, the :method: directive will be ignored if there is a
+# standard rdocable item following it.
+
+class RDoc::Parser::Ruby < RDoc::Parser
+
+ parse_files_matching(/\.rbw?$/)
+
+ include RDoc::RubyToken
+ include RDoc::TokenStream
+ include RDoc::Parser::RubyTools
+
+ ##
+ # RDoc::NormalClass type
+
+ NORMAL = "::"
+
+ ##
+ # RDoc::SingleClass type
+
+ SINGLE = "<<"
+
+ ##
+ # Creates a new Ruby parser.
+
+ def initialize(top_level, file_name, content, options, stats)
+ super
+
+ @size = 0
+ @token_listeners = nil
+ @scanner = RDoc::RubyLex.new content, @options
+ @scanner.exception_on_syntax_error = false
+ @prev_seek = nil
+ @markup = @options.markup
+ @track_visibility = :nodoc != @options.visibility
+
+ @encoding = nil
+ @encoding = @options.encoding if Object.const_defined? :Encoding
+
+ reset
+ end
+
+ ##
+ # Retrieves the read token stream and replaces +pattern+ with +replacement+
+ # using gsub. If the result is only a ";" returns an empty string.
+
+ def get_tkread_clean pattern, replacement # :nodoc:
+ read = get_tkread.gsub(pattern, replacement).strip
+ return '' if read == ';'
+ read
+ end
+
+ ##
+ # Extracts the visibility information for the visibility token +tk+
+ # and +single+ class type identifier.
+ #
+ # Returns the visibility type (a string), the visibility (a symbol) and
+ # +singleton+ if the methods following should be converted to singleton
+ # methods.
+
+ def get_visibility_information tk, single # :nodoc:
+ vis_type = tk.name
+ singleton = single == SINGLE
+
+ vis =
+ case vis_type
+ when 'private' then :private
+ when 'protected' then :protected
+ when 'public' then :public
+ when 'private_class_method' then
+ singleton = true
+ :private
+ when 'public_class_method' then
+ singleton = true
+ :public
+ when 'module_function' then
+ singleton = true
+ :public
+ else
+ raise RDoc::Error, "Invalid visibility: #{tk.name}"
+ end
+
+ return vis_type, vis, singleton
+ end
+
+ ##
+ # Look for the first comment in a file that isn't a shebang line.
+
+ def collect_first_comment
+ skip_tkspace
+ comment = ''
+ comment.force_encoding @encoding if @encoding
+ first_line = true
+ first_comment_tk_class = nil
+
+ tk = get_tk
+
+ while TkCOMMENT === tk
+ if first_line and tk.text =~ /\A#!/ then
+ skip_tkspace
+ tk = get_tk
+ elsif first_line and tk.text =~ /\A#\s*-\*-/ then
+ first_line = false
+ skip_tkspace
+ tk = get_tk
+ else
+ break if first_comment_tk_class and not first_comment_tk_class === tk
+ first_comment_tk_class = tk.class
+
+ first_line = false
+ comment << tk.text << "\n"
+ tk = get_tk
+
+ if TkNL === tk then
+ skip_tkspace false
+ tk = get_tk
+ end
+ end
+ end
+
+ unget_tk tk
+
+ new_comment comment
+ end
+
+ ##
+ # Consumes trailing whitespace from the token stream
+
+ def consume_trailing_spaces # :nodoc:
+ get_tkread
+ skip_tkspace false
+ end
+
+ ##
+ # Creates a new attribute in +container+ with +name+.
+
+ def create_attr container, single, name, rw, comment # :nodoc:
+ att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+ record_location att
+
+ container.add_attribute att
+ @stats.add_attribute att
+
+ att
+ end
+
+ ##
+ # Creates a module alias in +container+ at +rhs_name+ (or at the top-level
+ # for "::") with the name from +constant+.
+
+ def create_module_alias container, constant, rhs_name # :nodoc:
+ mod = if rhs_name =~ /^::/ then
+ @store.find_class_or_module rhs_name
+ else
+ container.find_module_named rhs_name
+ end
+
+ container.add_module_alias mod, constant.name, @top_level if mod
+ end
+
+ ##
+ # Aborts with +msg+
+
+ def error(msg)
+ msg = make_message msg
+
+ abort msg
+ end
+
+ ##
+ # Looks for a true or false token. Returns false if TkFALSE or TkNIL are
+ # found.
+
+ def get_bool
+ skip_tkspace
+ tk = get_tk
+ case tk
+ when TkTRUE
+ true
+ when TkFALSE, TkNIL
+ false
+ else
+ unget_tk tk
+ true
+ end
+ end
+
+ ##
+ # Look for the name of a class of module (optionally with a leading :: or
+ # with :: separated named) and return the ultimate name, the associated
+ # container, and the given name (with the ::).
+
+ def get_class_or_module container, ignore_constants = false
+ skip_tkspace
+ name_t = get_tk
+ given_name = ''
+
+ # class ::A -> A is in the top level
+ case name_t
+ when TkCOLON2, TkCOLON3 then # bug
+ name_t = get_tk
+ container = @top_level
+ given_name << '::'
+ end
+
+ skip_tkspace false
+ given_name << name_t.name
+
+ while TkCOLON2 === peek_tk do
+ prev_container = container
+ container = container.find_module_named name_t.name
+ container ||=
+ if ignore_constants then
+ RDoc::Context.new
+ else
+ c = prev_container.add_module RDoc::NormalModule, name_t.name
+ c.ignore unless prev_container.document_children
+ @top_level.add_to_classes_or_modules c
+ c
+ end
+
+ record_location container
+
+ get_tk
+ skip_tkspace false
+ name_t = get_tk
+ given_name << '::' << name_t.name
+ end
+
+ skip_tkspace false
+
+ return [container, name_t, given_name]
+ end
+
+ ##
+ # Return a superclass, which can be either a constant of an expression
+
+ def get_class_specification
+ case peek_tk
+ when TkSELF then return 'self'
+ when TkGVAR then return ''
+ end
+
+ res = get_constant
+
+ skip_tkspace false
+
+ get_tkread # empty out read buffer
+
+ tk = get_tk
+
+ case tk
+ when TkNL, TkCOMMENT, TkSEMICOLON then
+ unget_tk(tk)
+ return res
+ end
+
+ res += parse_call_parameters(tk)
+ res
+ end
+
+ ##
+ # Parse a constant, which might be qualified by one or more class or module
+ # names
+
+ def get_constant
+ res = ""
+ skip_tkspace false
+ tk = get_tk
+
+ while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
+ res += tk.name
+ tk = get_tk
+ end
+
+ unget_tk(tk)
+ res
+ end
+
+ ##
+ # Get a constant that may be surrounded by parens
+
+ def get_constant_with_optional_parens
+ skip_tkspace false
+
+ nest = 0
+
+ while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do
+ get_tk
+ skip_tkspace
+ nest += 1
+ end
+
+ name = get_constant
+
+ while nest > 0
+ skip_tkspace
+ tk = get_tk
+ nest -= 1 if TkRPAREN === tk
+ end
+
+ name
+ end
+
+ ##
+ # Little hack going on here. In the statement:
+ #
+ # f = 2*(1+yield)
+ #
+ # We see the RPAREN as the next token, so we need to exit early. This still
+ # won't catch all cases (such as "a = yield + 1"
+
+ def get_end_token tk # :nodoc:
+ case tk
+ when TkLPAREN, TkfLPAREN
+ TkRPAREN
+ when TkRPAREN
+ nil
+ else
+ TkNL
+ end
+ end
+
+ ##
+ # Retrieves the method container for a singleton method.
+
+ def get_method_container container, name_t # :nodoc:
+ prev_container = container
+ container = container.find_module_named(name_t.name)
+
+ unless container then
+ constant = prev_container.constants.find do |const|
+ const.name == name_t.name
+ end
+
+ if constant then
+ parse_method_dummy prev_container
+ return
+ end
+ end
+
+ unless container then
+ # TODO seems broken, should starting at Object in @store
+ obj = name_t.name.split("::").inject(Object) do |state, item|
+ state.const_get(item)
+ end rescue nil
+
+ type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
+
+ unless [Class, Module].include?(obj.class) then
+ warn("Couldn't find #{name_t.name}. Assuming it's a module")
+ end
+
+ if type == RDoc::NormalClass then
+ sclass = obj.superclass ? obj.superclass.name : nil
+ container = prev_container.add_class type, name_t.name, sclass
+ else
+ container = prev_container.add_module type, name_t.name
+ end
+
+ record_location container
+ end
+
+ container
+ end
+
+ ##
+ # Extracts a name or symbol from the token stream.
+
+ def get_symbol_or_name
+ tk = get_tk
+ case tk
+ when TkSYMBOL then
+ text = tk.text.sub(/^:/, '')
+
+ if TkASSIGN === peek_tk then
+ get_tk
+ text << '='
+ end
+
+ text
+ when TkId, TkOp then
+ tk.name
+ when TkAMPER,
+ TkDSTRING,
+ TkSTAR,
+ TkSTRING then
+ tk.text
+ else
+ raise RDoc::Error, "Name or symbol expected (got #{tk})"
+ end
+ end
+
+ def stop_at_EXPR_END # :nodoc:
+ @scanner.lex_state == :EXPR_END || !@scanner.continue
+ end
+
+ ##
+ # Marks containers between +container+ and +ancestor+ as ignored
+
+ def suppress_parents container, ancestor # :nodoc:
+ while container and container != ancestor do
+ container.suppress unless container.documented?
+ container = container.parent
+ end
+ end
+
+ ##
+ # Look for directives in a normal comment block:
+ #
+ # # :stopdoc:
+ # # Don't display comment from this point forward
+ #
+ # This routine modifies its +comment+ parameter.
+
+ def look_for_directives_in context, comment
+ @preprocess.handle comment, context do |directive, param|
+ case directive
+ when 'method', 'singleton-method',
+ 'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
+ false # handled elsewhere
+ when 'section' then
+ context.set_current_section param, comment.dup
+ comment.text = ''
+ break
+ end
+ end
+
+ remove_private_comments comment
+ end
+
+ ##
+ # Adds useful info about the parser to +message+
+
+ def make_message message
+ prefix = "#{@file_name}:"
+
+ prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner
+
+ "#{prefix} #{message}"
+ end
+
+ ##
+ # Creates a comment with the correct format
+
+ def new_comment comment
+ c = RDoc::Comment.new comment, @top_level
+ c.format = @markup
+ c
+ end
+
+ ##
+ # Creates an RDoc::Attr for the name following +tk+, setting the comment to
+ # +comment+.
+
+ def parse_attr(context, single, tk, comment)
+ offset = tk.seek
+ line_no = tk.line_no
+
+ args = parse_symbol_arg 1
+ if args.size > 0 then
+ name = args[0]
+ rw = "R"
+ skip_tkspace false
+ tk = get_tk
+
+ if TkCOMMA === tk then
+ rw = "RW" if get_bool
+ else
+ unget_tk tk
+ end
+
+ att = create_attr context, single, name, rw, comment
+ att.offset = offset
+ att.line = line_no
+
+ read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
+ else
+ warn "'attr' ignored - looks like a variable"
+ end
+ end
+
+ ##
+ # Creates an RDoc::Attr for each attribute listed after +tk+, setting the
+ # comment for each to +comment+.
+
+ def parse_attr_accessor(context, single, tk, comment)
+ offset = tk.seek
+ line_no = tk.line_no
+
+ args = parse_symbol_arg
+ rw = "?"
+
+ tmp = RDoc::CodeObject.new
+ read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
+ # TODO In most other places we let the context keep track of document_self
+ # and add found items appropriately but here we do not. I'm not sure why.
+ return if @track_visibility and not tmp.document_self
+
+ case tk.name
+ when "attr_reader" then rw = "R"
+ when "attr_writer" then rw = "W"
+ when "attr_accessor" then rw = "RW"
+ else
+ rw = '?'
+ end
+
+ for name in args
+ att = create_attr context, single, name, rw, comment
+ att.offset = offset
+ att.line = line_no
+ end
+ end
+
+ ##
+ # Parses an +alias+ in +context+ with +comment+
+
+ def parse_alias(context, single, tk, comment)
+ offset = tk.seek
+ line_no = tk.line_no
+
+ skip_tkspace
+
+ if TkLPAREN === peek_tk then
+ get_tk
+ skip_tkspace
+ end
+
+ new_name = get_symbol_or_name
+
+ @scanner.lex_state = :EXPR_FNAME
+
+ skip_tkspace
+ if TkCOMMA === peek_tk then
+ get_tk
+ skip_tkspace
+ end
+
+ begin
+ old_name = get_symbol_or_name
+ rescue RDoc::Error
+ return
+ end
+
+ al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
+ single == SINGLE)
+ record_location al
+ al.offset = offset
+ al.line = line_no
+
+ read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
+ context.add_alias al
+ @stats.add_alias al
+
+ al
+ end
+
+ ##
+ # Extracts call parameters from the token stream.
+
+ def parse_call_parameters(tk)
+ end_token = case tk
+ when TkLPAREN, TkfLPAREN
+ TkRPAREN
+ when TkRPAREN
+ return ""
+ else
+ TkNL
+ end
+ nest = 0
+
+ loop do
+ case tk
+ when TkSEMICOLON
+ break
+ when TkLPAREN, TkfLPAREN
+ nest += 1
+ when end_token
+ if end_token == TkRPAREN
+ nest -= 1
+ break if @scanner.lex_state == :EXPR_END and nest <= 0
+ else
+ break unless @scanner.continue
+ end
+ when TkCOMMENT, TkASSIGN, TkOPASGN
+ unget_tk(tk)
+ break
+ when nil then
+ break
+ end
+ tk = get_tk
+ end
+
+ get_tkread_clean "\n", " "
+ end
+
+ ##
+ # Parses a class in +context+ with +comment+
+
+ def parse_class container, single, tk, comment
+ offset = tk.seek
+ line_no = tk.line_no
+
+ declaration_context = container
+ container, name_t, given_name = get_class_or_module container
+
+ cls =
+ case name_t
+ when TkCONSTANT
+ parse_class_regular container, declaration_context, single,
+ name_t, given_name, comment
+ when TkLSHFT
+ case name = get_class_specification
+ when 'self', container.name
+ parse_statements container, SINGLE
+ return # don't update offset or line
+ else
+ parse_class_singleton container, name, comment
+ end
+ else
+ warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}"
+ return
+ end
+
+ cls.offset = offset
+ cls.line = line_no
+
+ cls
+ end
+
+ ##
+ # Parses and creates a regular class
+
+ def parse_class_regular container, declaration_context, single, # :nodoc:
+ name_t, given_name, comment
+ superclass = '::Object'
+
+ if given_name =~ /^::/ then
+ declaration_context = @top_level
+ given_name = $'
+ end
+
+ if TkLT === peek_tk then
+ get_tk
+ skip_tkspace
+ superclass = get_class_specification
+ superclass = '(unknown)' if superclass.empty?
+ end
+
+ cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
+ cls = declaration_context.add_class cls_type, given_name, superclass
+ cls.ignore unless container.document_children
+
+ read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+ record_location cls
+
+ cls.add_comment comment, @top_level
+
+ @top_level.add_to_classes_or_modules cls
+ @stats.add_class cls
+
+ suppress_parents container, declaration_context unless cls.document_self
+
+ parse_statements cls
+
+ cls
+ end
+
+ ##
+ # Parses a singleton class in +container+ with the given +name+ and
+ # +comment+.
+
+ def parse_class_singleton container, name, comment # :nodoc:
+ other = @store.find_class_named name
+
+ unless other then
+ if name =~ /^::/ then
+ name = $'
+ container = @top_level
+ end
+
+ other = container.add_module RDoc::NormalModule, name
+ record_location other
+
+ # class << $gvar
+ other.ignore if name.empty?
+
+ other.add_comment comment, @top_level
+ end
+
+ # notify :nodoc: all if not a constant-named class/module
+ # (and remove any comment)
+ unless name =~ /\A(::)?[A-Z]/ then
+ other.document_self = nil
+ other.document_children = false
+ other.clear_comment
+ end
+
+ @top_level.add_to_classes_or_modules other
+ @stats.add_class other
+
+ read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
+ parse_statements(other, SINGLE)
+
+ other
+ end
+
+ ##
+ # Parses a constant in +context+ with +comment+. If +ignore_constants+ is
+ # true, no found constants will be added to RDoc.
+
+ def parse_constant container, tk, comment, ignore_constants = false
+ offset = tk.seek
+ line_no = tk.line_no
+
+ name = tk.name
+ skip_tkspace false
+
+ return unless name =~ /^\w+$/
+
+ eq_tk = get_tk
+
+ if TkCOLON2 === eq_tk then
+ unget_tk eq_tk
+ unget_tk tk
+
+ container, name_t, = get_class_or_module container, ignore_constants
+
+ name = name_t.name
+
+ eq_tk = get_tk
+ end
+
+ unless TkASSIGN === eq_tk then
+ unget_tk eq_tk
+ return false
+ end
+
+ if TkGT === peek_tk then
+ unget_tk eq_tk
+ return
+ end
+
+ value = ''
+ con = RDoc::Constant.new name, value, comment
+
+ body = parse_constant_body container, con
+
+ return unless body
+
+ value.replace body
+ record_location con
+ con.offset = offset
+ con.line = line_no
+ read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
+
+ @stats.add_constant con
+ container.add_constant con
+
+ true
+ end
+
+ def parse_constant_body container, constant # :nodoc:
+ nest = 0
+ rhs_name = ''
+
+ get_tkread
+
+ tk = get_tk
+
+ loop do
+ case tk
+ when TkSEMICOLON then
+ break if nest <= 0
+ when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK,
+ TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then
+ nest += 1
+ when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
+ nest -= 1
+ when TkCOMMENT then
+ if nest <= 0 and stop_at_EXPR_END then
+ unget_tk tk
+ break
+ else
+ unget_tk tk
+ read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
+ end
+ when TkCONSTANT then
+ rhs_name << tk.name
+
+ if nest <= 0 and TkNL === peek_tk then
+ create_module_alias container, constant, rhs_name
+ break
+ end
+ when TkNL then
+ if nest <= 0 and stop_at_EXPR_END then
+ unget_tk tk
+ break
+ end
+ when TkCOLON2, TkCOLON3 then
+ rhs_name << '::'
+ when nil then
+ break
+ end
+ tk = get_tk
+ end
+
+ get_tkread_clean(/^[ \t]+/, '')
+ end
+
+ ##
+ # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for
+ # :method: or :attr: directives in +comment+.
+
+ def parse_comment container, tk, comment
+ return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
+ column = tk.char_no
+ offset = tk.seek
+ line_no = tk.line_no
+
+ text = comment.text
+
+ singleton = !!text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+
+ co =
+ if text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+ parse_comment_ghost container, text, $1, column, line_no, comment
+ elsif text.sub!(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+ parse_comment_attr container, $1, $3, comment
+ end
+
+ if co then
+ co.singleton = singleton
+ co.offset = offset
+ co.line = line_no
+ end
+
+ true
+ end
+
+ ##
+ # Parse a comment that is describing an attribute in +container+ with the
+ # given +name+ and +comment+.
+
+ def parse_comment_attr container, type, name, comment # :nodoc:
+ return if name.empty?
+
+ rw = case type
+ when 'attr_reader' then 'R'
+ when 'attr_writer' then 'W'
+ else 'RW'
+ end
+
+ create_attr container, NORMAL, name, rw, comment
+ end
+
+ def parse_comment_ghost container, text, name, column, line_no, # :nodoc:
+ comment
+ name = nil if name.empty?
+
+ meth = RDoc::GhostMethod.new get_tkread, name
+ record_location meth
+
+ meth.start_collecting_tokens
+ indent = TkSPACE.new 0, 1, 1
+ indent.set_text " " * column
+
+ position_comment = TkCOMMENT.new 0, line_no, 1
+ position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
+ meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+
+ meth.params =
+ if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then
+ $1
+ else
+ ''
+ end
+
+ comment.normalize
+ comment.extract_call_seq meth
+
+ return unless meth.name
+
+ container.add_method meth
+
+ meth.comment = comment
+
+ @stats.add_method meth
+
+ meth
+ end
+
+ ##
+ # Creates an RDoc::Method on +container+ from +comment+ if there is a
+ # Signature section in the comment
+
+ def parse_comment_tomdoc container, tk, comment
+ return unless signature = RDoc::TomDoc.signature(comment)
+ offset = tk.seek
+ line_no = tk.line_no
+
+ name, = signature.split %r%[ \(]%, 2
+
+ meth = RDoc::GhostMethod.new get_tkread, name
+ record_location meth
+ meth.offset = offset
+ meth.line = line_no
+
+ meth.start_collecting_tokens
+ indent = TkSPACE.new 0, 1, 1
+ indent.set_text " " * offset
+
+ position_comment = TkCOMMENT.new 0, line_no, 1
+ position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
+ meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+
+ meth.call_seq = signature
+
+ comment.normalize
+
+ return unless meth.name
+
+ container.add_method meth
+
+ meth.comment = comment
+
+ @stats.add_method meth
+ end
+
+ ##
+ # Parses an +include+ or +extend+, indicated by the +klass+ and adds it to
+ # +container+ # with +comment+
+
+ def parse_extend_or_include klass, container, comment # :nodoc:
+ loop do
+ skip_tkspace_comment
+
+ name = get_constant_with_optional_parens
+
+ unless name.empty? then
+ obj = container.add klass, name, comment
+ record_location obj
+ end
+
+ return unless TkCOMMA === peek_tk
+
+ get_tk
+ end
+ end
+
+ ##
+ # Parses identifiers that can create new methods or change visibility.
+ #
+ # Returns true if the comment was not consumed.
+
+ def parse_identifier container, single, tk, comment # :nodoc:
+ case tk.name
+ when 'private', 'protected', 'public', 'private_class_method',
+ 'public_class_method', 'module_function' then
+ parse_visibility container, single, tk
+ return true
+ when 'attr' then
+ parse_attr container, single, tk, comment
+ when /^attr_(reader|writer|accessor)$/ then
+ parse_attr_accessor container, single, tk, comment
+ when 'alias_method' then
+ parse_alias container, single, tk, comment
+ when 'require', 'include' then
+ # ignore
+ else
+ if comment.text =~ /\A#\#$/ then
+ case comment.text
+ when /^# +:?attr(_reader|_writer|_accessor)?:/ then
+ parse_meta_attr container, single, tk, comment
+ else
+ method = parse_meta_method container, single, tk, comment
+ method.params = container.params if
+ container.params
+ method.block_params = container.block_params if
+ container.block_params
+ end
+ end
+ end
+
+ false
+ end
+
+ ##
+ # Parses a meta-programmed attribute and creates an RDoc::Attr.
+ #
+ # To create foo and bar attributes on class C with comment "My attributes":
+ #
+ # class C
+ #
+ # ##
+ # # :attr:
+ # #
+ # # My attributes
+ #
+ # my_attr :foo, :bar
+ #
+ # end
+ #
+ # To create a foo attribute on class C with comment "My attribute":
+ #
+ # class C
+ #
+ # ##
+ # # :attr: foo
+ # #
+ # # My attribute
+ #
+ # my_attr :foo, :bar
+ #
+ # end
+
+ def parse_meta_attr(context, single, tk, comment)
+ args = parse_symbol_arg
+ rw = "?"
+
+ # If nodoc is given, don't document any of them
+
+ tmp = RDoc::CodeObject.new
+ read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
+
+ if comment.text.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+ rw = case $1
+ when 'attr_reader' then 'R'
+ when 'attr_writer' then 'W'
+ else 'RW'
+ end
+ name = $3 unless $3.empty?
+ end
+
+ if name then
+ att = create_attr context, single, name, rw, comment
+ else
+ args.each do |attr_name|
+ att = create_attr context, single, attr_name, rw, comment
+ end
+ end
+
+ att
+ end
+
+ ##
+ # Parses a meta-programmed method
+
+ def parse_meta_method(container, single, tk, comment)
+ column = tk.char_no
+ offset = tk.seek
+ line_no = tk.line_no
+
+ start_collecting_tokens
+ add_token tk
+ add_token_listener self
+
+ skip_tkspace false
+
+ singleton = !!comment.text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+
+ name = parse_meta_method_name comment, tk
+
+ return unless name
+
+ meth = RDoc::MetaMethod.new get_tkread, name
+ record_location meth
+ meth.offset = offset
+ meth.line = line_no
+ meth.singleton = singleton
+
+ remove_token_listener self
+
+ meth.start_collecting_tokens
+ indent = TkSPACE.new 0, 1, 1
+ indent.set_text " " * column
+
+ position_comment = TkCOMMENT.new 0, line_no, 1
+ position_comment.value = "# File #{@top_level.relative_name}, line #{line_no}"
+ meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+ meth.add_tokens @token_stream
+
+ parse_meta_method_params container, single, meth, tk, comment
+
+ meth.comment = comment
+
+ @stats.add_method meth
+
+ meth
+ end
+
+ ##
+ # Parses the name of a metaprogrammed method. +comment+ is used to
+ # determine the name while +tk+ is used in an error message if the name
+ # cannot be determined.
+
+ def parse_meta_method_name comment, tk # :nodoc:
+ if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+ return $1 unless $1.empty?
+ end
+
+ name_t = get_tk
+
+ case name_t
+ when TkSYMBOL then
+ name_t.text[1..-1]
+ when TkSTRING then
+ name_t.value[1..-2]
+ when TkASSIGN then # ignore
+ remove_token_listener self
+
+ nil
+ else
+ warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'"
+ 'unknown'
+ end
+ end
+
+ ##
+ # Parses the parameters and block for a meta-programmed method.
+
+ def parse_meta_method_params container, single, meth, tk, comment # :nodoc:
+ token_listener meth do
+ meth.params = ''
+
+ comment.normalize
+ comment.extract_call_seq meth
+
+ container.add_method meth
+
+ last_tk = tk
+
+ while tk = get_tk do
+ case tk
+ when TkSEMICOLON then
+ break
+ when TkNL then
+ break unless last_tk and TkCOMMA === last_tk
+ when TkSPACE then
+ # expression continues
+ when TkDO then
+ parse_statements container, single, meth
+ break
+ else
+ last_tk = tk
+ end
+ end
+ end
+ end
+
+ ##
+ # Parses a normal method defined by +def+
+
+ def parse_method(container, single, tk, comment)
+ singleton = nil
+ added_container = false
+ name = nil
+ column = tk.char_no
+ offset = tk.seek
+ line_no = tk.line_no
+
+ start_collecting_tokens
+ add_token tk
+
+ token_listener self do
+ prev_container = container
+ name, container, singleton = parse_method_name container
+ added_container = container != prev_container
+ end
+
+ return unless name
+
+ meth = RDoc::AnyMethod.new get_tkread, name
+ meth.singleton = single == SINGLE ? true : singleton
+
+ record_location meth
+ meth.offset = offset
+ meth.line = line_no
+
+ meth.start_collecting_tokens
+ indent = TkSPACE.new 0, 1, 1
+ indent.set_text " " * column
+
+ token = TkCOMMENT.new 0, line_no, 1
+ token.set_text "# File #{@top_level.relative_name}, line #{line_no}"
+ meth.add_tokens [token, NEWLINE_TOKEN, indent]
+ meth.add_tokens @token_stream
+
+ parse_method_params_and_body container, single, meth, added_container
+
+ comment.normalize
+ comment.extract_call_seq meth
+
+ meth.comment = comment
+
+ @stats.add_method meth
+ end
+
+ ##
+ # Parses the parameters and body of +meth+
+
+ def parse_method_params_and_body container, single, meth, added_container
+ token_listener meth do
+ @scanner.continue = false
+ parse_method_parameters meth
+
+ if meth.document_self or not @track_visibility then
+ container.add_method meth
+ elsif added_container then
+ container.document_self = false
+ end
+
+ # Having now read the method parameters and documentation modifiers, we
+ # now know whether we have to rename #initialize to ::new
+
+ if meth.name == "initialize" && !meth.singleton then
+ if meth.dont_rename_initialize then
+ meth.visibility = :protected
+ else
+ meth.singleton = true
+ meth.name = "new"
+ meth.visibility = :public
+ end
+ end
+
+ parse_statements container, single, meth
+ end
+ end
+
+ ##
+ # Parses a method that needs to be ignored.
+
+ def parse_method_dummy container
+ dummy = RDoc::Context.new
+ dummy.parent = container
+ dummy.store = container.store
+ skip_method dummy
+ end
+
+ ##
+ # Parses the name of a method in +container+.
+ #
+ # Returns the method name, the container it is in (for def Foo.name) and if
+ # it is a singleton or regular method.
+
+ def parse_method_name container # :nodoc:
+ @scanner.lex_state = :EXPR_FNAME
+
+ skip_tkspace
+ name_t = get_tk
+ back_tk = skip_tkspace
+ singleton = false
+
+ case dot = get_tk
+ when TkDOT, TkCOLON2 then
+ singleton = true
+
+ name, container = parse_method_name_singleton container, name_t
+ else
+ unget_tk dot
+ back_tk.reverse_each do |token|
+ unget_tk token
+ end
+
+ name = parse_method_name_regular container, name_t
+ end
+
+ return name, container, singleton
+ end
+
+ ##
+ # For the given +container+ and initial name token +name_t+ the method name
+ # is parsed from the token stream for a regular method.
+
+ def parse_method_name_regular container, name_t # :nodoc:
+ case name_t
+ when TkSTAR, TkAMPER then
+ name_t.text
+ else
+ unless name_t.respond_to? :name then
+ warn "expected method name token, . or ::, got #{name_t.inspect}"
+ skip_method container
+ return
+ end
+ name_t.name
+ end
+ end
+
+ ##
+ # For the given +container+ and initial name token +name_t+ the method name
+ # and the new +container+ (if necessary) are parsed from the token stream
+ # for a singleton method.
+
+ def parse_method_name_singleton container, name_t # :nodoc:
+ @scanner.lex_state = :EXPR_FNAME
+ skip_tkspace
+ name_t2 = get_tk
+
+ name =
+ case name_t
+ when TkSELF, TkMOD then
+ case name_t2
+ # NOTE: work around '[' being consumed early and not being re-tokenized
+ # as a TkAREF
+ when TkfLBRACK then
+ get_tk
+ '[]'
+ else
+ name_t2.name
+ end
+ when TkCONSTANT then
+ name = name_t2.name
+
+ container = get_method_container container, name_t
+
+ return unless container
+
+ name
+ when TkIDENTIFIER, TkIVAR, TkGVAR then
+ parse_method_dummy container
+
+ nil
+ when TkTRUE, TkFALSE, TkNIL then
+ klass_name = "#{name_t.name.capitalize}Class"
+ container = @store.find_class_named klass_name
+ container ||= @top_level.add_class RDoc::NormalClass, klass_name
+
+ name_t2.name
+ else
+ warn "unexpected method name token #{name_t.inspect}"
+ # break
+ skip_method container
+
+ nil
+ end
+
+ return name, container
+ end
+
+ ##
+ # Extracts +yield+ parameters from +method+
+
+ def parse_method_or_yield_parameters(method = nil,
+ modifiers = RDoc::METHOD_MODIFIERS)
+ skip_tkspace false
+ tk = get_tk
+ end_token = get_end_token tk
+ return '' unless end_token
+
+ nest = 0
+
+ loop do
+ case tk
+ when TkSEMICOLON then
+ break if nest == 0
+ when TkLBRACE, TkfLBRACE then
+ nest += 1
+ when TkRBRACE then
+ nest -= 1
+ if nest <= 0
+ # we might have a.each { |i| yield i }
+ unget_tk(tk) if nest < 0
+ break
+ end
+ when TkLPAREN, TkfLPAREN then
+ nest += 1
+ when end_token then
+ if end_token == TkRPAREN
+ nest -= 1
+ break if nest <= 0
+ else
+ break unless @scanner.continue
+ end
+ when TkRPAREN then
+ nest -= 1
+ when method && method.block_params.nil? && TkCOMMENT then
+ unget_tk tk
+ read_documentation_modifiers method, modifiers
+ @read.pop
+ when TkCOMMENT then
+ @read.pop
+ when nil then
+ break
+ end
+ tk = get_tk
+ end
+
+ get_tkread_clean(/\s+/, ' ')
+ end
+
+ ##
+ # Capture the method's parameters. Along the way, look for a comment
+ # containing:
+ #
+ # # yields: ....
+ #
+ # and add this as the block_params for the method
+
+ def parse_method_parameters method
+ res = parse_method_or_yield_parameters method
+
+ res = "(#{res})" unless res =~ /\A\(/
+ method.params = res unless method.params
+
+ return if method.block_params
+
+ skip_tkspace false
+ read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
+ end
+
+ ##
+ # Parses an RDoc::NormalModule in +container+ with +comment+
+
+ def parse_module container, single, tk, comment
+ container, name_t, = get_class_or_module container
+
+ name = name_t.name
+
+ mod = container.add_module RDoc::NormalModule, name
+ mod.ignore unless container.document_children
+ record_location mod
+
+ read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+ mod.add_comment comment, @top_level
+ parse_statements mod
+
+ @stats.add_module mod
+ end
+
+ ##
+ # Parses an RDoc::Require in +context+ containing +comment+
+
+ def parse_require(context, comment)
+ skip_tkspace_comment
+ tk = get_tk
+
+ if TkLPAREN === tk then
+ skip_tkspace_comment
+ tk = get_tk
+ end
+
+ name = tk.text if TkSTRING === tk
+
+ if name then
+ @top_level.add_require RDoc::Require.new(name, comment)
+ else
+ unget_tk tk
+ end
+ end
+
+ ##
+ # Parses a rescue
+
+ def parse_rescue
+ skip_tkspace false
+
+ while tk = get_tk
+ case tk
+ when TkNL, TkSEMICOLON then
+ break
+ when TkCOMMA then
+ skip_tkspace false
+
+ get_tk if TkNL === peek_tk
+ end
+
+ skip_tkspace false
+ end
+ end
+
+ ##
+ # The core of the Ruby parser.
+
+ def parse_statements(container, single = NORMAL, current_method = nil,
+ comment = new_comment(''))
+ raise 'no' unless RDoc::Comment === comment
+ comment.force_encoding @encoding if @encoding
+
+ nest = 1
+ save_visibility = container.visibility
+
+ non_comment_seen = true
+
+ while tk = get_tk do
+ keep_comment = false
+ try_parse_comment = false
+
+ non_comment_seen = true unless TkCOMMENT === tk
+
+ case tk
+ when TkNL then
+ skip_tkspace
+ tk = get_tk
+
+ if TkCOMMENT === tk then
+ if non_comment_seen then
+ # Look for RDoc in a comment about to be thrown away
+ non_comment_seen = parse_comment container, tk, comment unless
+ comment.empty?
+
+ comment = ''
+ comment.force_encoding @encoding if @encoding
+ end
+
+ while TkCOMMENT === tk do
+ comment << tk.text << "\n"
+
+ tk = get_tk
+
+ if TkNL === tk then
+ skip_tkspace false # leading spaces
+ tk = get_tk
+ end
+ end
+
+ comment = new_comment comment
+
+ unless comment.empty? then
+ look_for_directives_in container, comment
+
+ if container.done_documenting then
+ throw :eof if RDoc::TopLevel === container
+ container.ongoing_visibility = save_visibility
+ end
+ end
+
+ keep_comment = true
+ else
+ non_comment_seen = true
+ end
+
+ unget_tk tk
+ keep_comment = true
+
+ when TkCLASS then
+ parse_class container, single, tk, comment
+
+ when TkMODULE then
+ parse_module container, single, tk, comment
+
+ when TkDEF then
+ parse_method container, single, tk, comment
+
+ when TkCONSTANT then
+ unless parse_constant container, tk, comment, current_method then
+ try_parse_comment = true
+ end
+
+ when TkALIAS then
+ parse_alias container, single, tk, comment unless current_method
+
+ when TkYIELD then
+ if current_method.nil? then
+ warn "Warning: yield outside of method" if container.document_self
+ else
+ parse_yield container, single, tk, current_method
+ end
+
+ # Until and While can have a 'do', which shouldn't increase the nesting.
+ # We can't solve the general case, but we can handle most occurrences by
+ # ignoring a do at the end of a line.
+
+ when TkUNTIL, TkWHILE then
+ nest += 1
+ skip_optional_do_after_expression
+
+ # 'for' is trickier
+ when TkFOR then
+ nest += 1
+ skip_for_variable
+ skip_optional_do_after_expression
+
+ when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then
+ nest += 1
+
+ when TkSUPER then
+ current_method.calls_super = true if current_method
+
+ when TkRESCUE then
+ parse_rescue
+
+ when TkIDENTIFIER then
+ if nest == 1 and current_method.nil? then
+ keep_comment = parse_identifier container, single, tk, comment
+ end
+
+ case tk.name
+ when "require" then
+ parse_require container, comment
+ when "include" then
+ parse_extend_or_include RDoc::Include, container, comment
+ when "extend" then
+ parse_extend_or_include RDoc::Extend, container, comment
+ end
+
+ when TkEND then
+ nest -= 1
+ if nest == 0 then
+ read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
+ container.ongoing_visibility = save_visibility
+
+ parse_comment container, tk, comment unless comment.empty?
+
+ return
+ end
+ else
+ try_parse_comment = nest == 1
+ end
+
+ if try_parse_comment then
+ non_comment_seen = parse_comment container, tk, comment unless
+ comment.empty?
+
+ keep_comment = false
+ end
+
+ unless keep_comment then
+ comment = new_comment ''
+ comment.force_encoding @encoding if @encoding
+ container.params = nil
+ container.block_params = nil
+ end
+
+ consume_trailing_spaces
+ end
+
+ container.params = nil
+ container.block_params = nil
+ end
+
+ ##
+ # Parse up to +no+ symbol arguments
+
+ def parse_symbol_arg(no = nil)
+ skip_tkspace_comment
+
+ case tk = get_tk
+ when TkLPAREN
+ parse_symbol_arg_paren no
+ else
+ parse_symbol_arg_space no, tk
+ end
+ end
+
+ ##
+ # Parses up to +no+ symbol arguments surrounded by () and places them in
+ # +args+.
+
+ def parse_symbol_arg_paren no # :nodoc:
+ args = []
+
+ loop do
+ skip_tkspace_comment
+ if tk1 = parse_symbol_in_arg
+ args.push tk1
+ break if no and args.size >= no
+ end
+
+ skip_tkspace_comment
+ case tk2 = get_tk
+ when TkRPAREN
+ break
+ when TkCOMMA
+ else
+ warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
+ break
+ end
+ end
+
+ args
+ end
+
+ ##
+ # Parses up to +no+ symbol arguments separated by spaces and places them in
+ # +args+.
+
+ def parse_symbol_arg_space no, tk # :nodoc:
+ args = []
+
+ unget_tk tk
+ if tk = parse_symbol_in_arg
+ args.push tk
+ return args if no and args.size >= no
+ end
+
+ loop do
+ skip_tkspace false
+
+ tk1 = get_tk
+ unless TkCOMMA === tk1 then
+ unget_tk tk1
+ break
+ end
+
+ skip_tkspace_comment
+ if tk = parse_symbol_in_arg
+ args.push tk
+ break if no and args.size >= no
+ end
+ end
+
+ args
+ end
+
+ ##
+ # Returns symbol text from the next token
+
+ def parse_symbol_in_arg
+ case tk = get_tk
+ when TkSYMBOL
+ tk.text.sub(/^:/, '')
+ when TkSTRING
+ eval @read[-1]
+ when TkDSTRING, TkIDENTIFIER then
+ nil # ignore
+ else
+ warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
+ nil
+ end
+ end
+
+ ##
+ # Parses statements in the top-level +container+
+
+ def parse_top_level_statements container
+ comment = collect_first_comment
+
+ look_for_directives_in container, comment
+
+ throw :eof if container.done_documenting
+
+ @markup = comment.format
+
+ # HACK move if to RDoc::Context#comment=
+ container.comment = comment if container.document_self unless comment.empty?
+
+ parse_statements container, NORMAL, nil, comment
+ end
+
+ ##
+ # Determines the visibility in +container+ from +tk+
+
+ def parse_visibility(container, single, tk)
+ vis_type, vis, singleton = get_visibility_information tk, single
+
+ skip_tkspace_comment false
+
+ case peek_tk
+ # Ryan Davis suggested the extension to ignore modifiers, because he
+ # often writes
+ #
+ # protected unless $TESTING
+ #
+ when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then
+ container.ongoing_visibility = vis
+ else
+ update_visibility container, vis_type, vis, singleton
+ end
+ end
+
+ ##
+ # Determines the block parameter for +context+
+
+ def parse_yield(context, single, tk, method)
+ return if method.block_params
+
+ get_tkread
+ @scanner.continue = false
+ method.block_params = parse_method_or_yield_parameters
+ end
+
+ ##
+ # Directives are modifier comments that can appear after class, module, or
+ # method names. For example:
+ #
+ # def fred # :yields: a, b
+ #
+ # or:
+ #
+ # class MyClass # :nodoc:
+ #
+ # We return the directive name and any parameters as a two element array if
+ # the name is in +allowed+. A directive can be found anywhere up to the end
+ # of the current line.
+
+ def read_directive allowed
+ tokens = []
+
+ while tk = get_tk do
+ tokens << tk
+
+ case tk
+ when TkNL, TkDEF then
+ return
+ when TkCOMMENT then
+ return unless tk.text =~ /\s*:?([\w-]+):\s*(.*)/
+
+ directive = $1.downcase
+
+ return [directive, $2] if allowed.include? directive
+
+ return
+ end
+ end
+ ensure
+ unless tokens.length == 1 and TkCOMMENT === tokens.first then
+ tokens.reverse_each do |token|
+ unget_tk token
+ end
+ end
+ end
+
+ ##
+ # Handles directives following the definition for +context+ (any
+ # RDoc::CodeObject) if the directives are +allowed+ at this point.
+ #
+ # See also RDoc::Markup::PreProcess#handle_directive
+
+ def read_documentation_modifiers context, allowed
+ directive, value = read_directive allowed
+
+ return unless directive
+
+ @preprocess.handle_directive '', directive, value, context do |dir, param|
+ if %w[notnew not_new not-new].include? dir then
+ context.dont_rename_initialize = true
+
+ true
+ end
+ end
+ end
+
+ ##
+ # Records the location of this +container+ in the file for this parser and
+ # adds it to the list of classes and modules in the file.
+
+ def record_location container # :nodoc:
+ case container
+ when RDoc::ClassModule then
+ @top_level.add_to_classes_or_modules container
+ end
+
+ container.record_location @top_level
+ end
+
+ ##
+ # Removes private comments from +comment+
+ #--
+ # TODO remove
+
+ def remove_private_comments comment
+ comment.remove_private
+ end
+
+ ##
+ # Scans this Ruby file for Ruby constructs
+
+ def scan
+ reset
+
+ catch :eof do
+ begin
+ parse_top_level_statements @top_level
+
+ rescue StandardError => e
+ bytes = ''
+
+ 20.times do @scanner.ungetc end
+ count = 0
+ 60.times do |i|
+ count = i
+ byte = @scanner.getc
+ break unless byte
+ bytes << byte
+ end
+ count -= 20
+ count.times do @scanner.ungetc end
+
+ $stderr.puts <<-EOF
+
+#{self.class} failure around line #{@scanner.line_no} of
+#{@file_name}
+
+ EOF
+
+ unless bytes.empty? then
+ $stderr.puts
+ $stderr.puts bytes.inspect
+ end
+
+ raise e
+ end
+ end
+
+ @top_level
+ end
+
+ ##
+ # while, until, and for have an optional do
+
+ def skip_optional_do_after_expression
+ skip_tkspace false
+ tk = get_tk
+ end_token = get_end_token tk
+
+ b_nest = 0
+ nest = 0
+ @scanner.continue = false
+
+ loop do
+ case tk
+ when TkSEMICOLON then
+ break if b_nest.zero?
+ when TkLPAREN, TkfLPAREN then
+ nest += 1
+ when TkBEGIN then
+ b_nest += 1
+ when TkEND then
+ b_nest -= 1
+ when TkDO
+ break if nest.zero?
+ when end_token then
+ if end_token == TkRPAREN
+ nest -= 1
+ break if @scanner.lex_state == :EXPR_END and nest.zero?
+ else
+ break unless @scanner.continue
+ end
+ when nil then
+ break
+ end
+ tk = get_tk
+ end
+
+ skip_tkspace false
+
+ get_tk if TkDO === peek_tk
+ end
+
+ ##
+ # skip the var [in] part of a 'for' statement
+
+ def skip_for_variable
+ skip_tkspace false
+ get_tk
+ skip_tkspace false
+ tk = get_tk
+ unget_tk(tk) unless TkIN === tk
+ end
+
+ ##
+ # Skips the next method in +container+
+
+ def skip_method container
+ meth = RDoc::AnyMethod.new "", "anon"
+ parse_method_parameters meth
+ parse_statements container, false, meth
+ end
+
+ ##
+ # Skip spaces until a comment is found
+
+ def skip_tkspace_comment(skip_nl = true)
+ loop do
+ skip_tkspace skip_nl
+ return unless TkCOMMENT === peek_tk
+ get_tk
+ end
+ end
+
+ ##
+ # Updates visibility in +container+ from +vis_type+ and +vis+.
+
+ def update_visibility container, vis_type, vis, singleton # :nodoc:
+ new_methods = []
+
+ case vis_type
+ when 'module_function' then
+ args = parse_symbol_arg
+ container.set_visibility_for args, :private, false
+
+ container.methods_matching args do |m|
+ s_m = m.dup
+ record_location s_m
+ s_m.singleton = true
+ new_methods << s_m
+ end
+ when 'public_class_method', 'private_class_method' then
+ args = parse_symbol_arg
+
+ container.methods_matching args, true do |m|
+ if m.parent != container then
+ m = m.dup
+ record_location m
+ new_methods << m
+ end
+
+ m.visibility = vis
+ end
+ else
+ args = parse_symbol_arg
+ container.set_visibility_for args, vis, singleton
+ end
+
+ new_methods.each do |method|
+ case method
+ when RDoc::AnyMethod then
+ container.add_method method
+ when RDoc::Attr then
+ container.add_attribute method
+ end
+ method.visibility = vis
+ end
+ end
+
+ ##
+ # Prints +message+ to +$stderr+ unless we're being quiet
+
+ def warn message
+ @options.warn make_message message
+ end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/ruby_tools.rb b/jni/ruby/lib/rdoc/parser/ruby_tools.rb
new file mode 100644
index 0000000..654431e
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/ruby_tools.rb
@@ -0,0 +1,167 @@
+##
+# Collection of methods for writing parsers against RDoc::RubyLex and
+# RDoc::RubyToken
+
+module RDoc::Parser::RubyTools
+
+ include RDoc::RubyToken
+
+ ##
+ # Adds a token listener +obj+, but you should probably use token_listener
+
+ def add_token_listener(obj)
+ @token_listeners ||= []
+ @token_listeners << obj
+ end
+
+ ##
+ # Fetches the next token from the scanner
+
+ def get_tk
+ tk = nil
+
+ if @tokens.empty? then
+ tk = @scanner.token
+ @read.push @scanner.get_readed
+ puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+ else
+ @read.push @unget_read.shift
+ tk = @tokens.shift
+ puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
+ end
+
+ tk = nil if TkEND_OF_SCRIPT === tk
+
+ if TkSYMBEG === tk then
+ set_token_position tk.line_no, tk.char_no
+
+ case tk1 = get_tk
+ when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
+ if tk1.respond_to?(:name) then
+ tk = Token(TkSYMBOL).set_text(":" + tk1.name)
+ else
+ tk = Token(TkSYMBOL).set_text(":" + tk1.text)
+ end
+
+ # remove the identifier we just read to replace it with a symbol
+ @token_listeners.each do |obj|
+ obj.pop_token
+ end if @token_listeners
+ else
+ tk = tk1
+ end
+ end
+
+ # inform any listeners of our shiny new token
+ @token_listeners.each do |obj|
+ obj.add_token(tk)
+ end if @token_listeners
+
+ tk
+ end
+
+ ##
+ # Reads and returns all tokens up to one of +tokens+. Leaves the matched
+ # token in the token list.
+
+ def get_tk_until(*tokens)
+ read = []
+
+ loop do
+ tk = get_tk
+
+ case tk
+ when *tokens then
+ unget_tk tk
+ break
+ end
+
+ read << tk
+ end
+
+ read
+ end
+
+ ##
+ # Retrieves a String representation of the read tokens
+
+ def get_tkread
+ read = @read.join("")
+ @read = []
+ read
+ end
+
+ ##
+ # Peek equivalent for get_tkread
+
+ def peek_read
+ @read.join('')
+ end
+
+ ##
+ # Peek at the next token, but don't remove it from the stream
+
+ def peek_tk
+ unget_tk(tk = get_tk)
+ tk
+ end
+
+ ##
+ # Removes the token listener +obj+
+
+ def remove_token_listener(obj)
+ @token_listeners.delete(obj)
+ end
+
+ ##
+ # Resets the tools
+
+ def reset
+ @read = []
+ @tokens = []
+ @unget_read = []
+ @nest = 0
+ end
+
+ ##
+ # Skips whitespace tokens including newlines if +skip_nl+ is true
+
+ def skip_tkspace(skip_nl = true) # HACK dup
+ tokens = []
+
+ while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
+ tokens.push tk
+ end
+
+ unget_tk tk
+ tokens
+ end
+
+ ##
+ # Has +obj+ listen to tokens
+
+ def token_listener(obj)
+ add_token_listener obj
+ yield
+ ensure
+ remove_token_listener obj
+ end
+
+ ##
+ # Returns +tk+ to the scanner
+
+ def unget_tk(tk)
+ @tokens.unshift tk
+ @unget_read.unshift @read.pop
+
+ # Remove this token from any listeners
+ @token_listeners.each do |obj|
+ obj.pop_token
+ end if @token_listeners
+
+ nil
+ end
+
+end
+
+
diff --git a/jni/ruby/lib/rdoc/parser/simple.rb b/jni/ruby/lib/rdoc/parser/simple.rb
new file mode 100644
index 0000000..65cfc1b
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/simple.rb
@@ -0,0 +1,61 @@
+##
+# Parse a non-source file. We basically take the whole thing as one big
+# comment.
+
+class RDoc::Parser::Simple < RDoc::Parser
+
+ include RDoc::Parser::Text
+
+ parse_files_matching(//)
+
+ attr_reader :content # :nodoc:
+
+ ##
+ # Prepare to parse a plain file
+
+ def initialize(top_level, file_name, content, options, stats)
+ super
+
+ preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
+
+ preprocess.handle @content, @top_level
+ end
+
+ ##
+ # Extract the file contents and attach them to the TopLevel as a comment
+
+ def scan
+ comment = remove_coding_comment @content
+ comment = remove_private_comment comment
+
+ comment = RDoc::Comment.new comment, @top_level
+
+ @top_level.comment = comment
+ @top_level
+ end
+
+ ##
+ # Removes the encoding magic comment from +text+
+
+ def remove_coding_comment text
+ text.sub(/\A# .*coding[=:].*$/, '')
+ end
+
+ ##
+ # Removes private comments.
+ #
+ # Unlike RDoc::Comment#remove_private this implementation only looks for two
+ # dashes at the beginning of the line. Three or more dashes are considered
+ # to be a rule and ignored.
+
+ def remove_private_comment comment
+ # Workaround for gsub encoding for Ruby 1.9.2 and earlier
+ empty = ''
+ empty.force_encoding comment.encoding if Object.const_defined? :Encoding
+
+ comment = comment.gsub(%r%^--\n.*?^\+\+\n?%m, empty)
+ comment.sub(%r%^--\n.*%m, empty)
+ end
+
+end
+
diff --git a/jni/ruby/lib/rdoc/parser/text.rb b/jni/ruby/lib/rdoc/parser/text.rb
new file mode 100644
index 0000000..f973313
--- /dev/null
+++ b/jni/ruby/lib/rdoc/parser/text.rb
@@ -0,0 +1,11 @@
+##
+# Indicates this parser is text and doesn't contain code constructs.
+#
+# Include this module in a RDoc::Parser subclass to make it show up as a file,
+# not as part of a class or module.
+#--
+# This is not named File to avoid overriding ::File
+
+module RDoc::Parser::Text
+end
+