diff options
Diffstat (limited to 'jni/ruby/lib/rdoc/parser')
| -rw-r--r-- | jni/ruby/lib/rdoc/parser/c.rb | 1229 | ||||
| -rw-r--r-- | jni/ruby/lib/rdoc/parser/changelog.rb | 198 | ||||
| -rw-r--r-- | jni/ruby/lib/rdoc/parser/markdown.rb | 23 | ||||
| -rw-r--r-- | jni/ruby/lib/rdoc/parser/rd.rb | 22 | ||||
| -rw-r--r-- | jni/ruby/lib/rdoc/parser/ruby.rb | 2160 | ||||
| -rw-r--r-- | jni/ruby/lib/rdoc/parser/ruby_tools.rb | 167 | ||||
| -rw-r--r-- | jni/ruby/lib/rdoc/parser/simple.rb | 61 | ||||
| -rw-r--r-- | jni/ruby/lib/rdoc/parser/text.rb | 11 |
8 files changed, 3871 insertions, 0 deletions
diff --git a/jni/ruby/lib/rdoc/parser/c.rb b/jni/ruby/lib/rdoc/parser/c.rb new file mode 100644 index 0000000..fd336f5 --- /dev/null +++ b/jni/ruby/lib/rdoc/parser/c.rb @@ -0,0 +1,1229 @@ +require 'tsort' + +## +# RDoc::Parser::C attempts to parse C extension files. It looks for +# the standard patterns that you find in extensions: <tt>rb_define_class, +# rb_define_method</tt> and so on. It tries to find the corresponding +# C source for the methods and extract comments, but if we fail +# we don't worry too much. +# +# The comments associated with a Ruby method are extracted from the C +# comment block associated with the routine that _implements_ that +# method, that is to say the method whose name is given in the +# <tt>rb_define_method</tt> call. For example, you might write: +# +# /* +# * Returns a new array that is a one-dimensional flattening of this +# * array (recursively). That is, for every element that is an array, +# * extract its elements into the new array. +# * +# * s = [ 1, 2, 3 ] #=> [1, 2, 3] +# * t = [ 4, 5, 6, [7, 8] ] #=> [4, 5, 6, [7, 8]] +# * a = [ s, t, 9, 10 ] #=> [[1, 2, 3], [4, 5, 6, [7, 8]], 9, 10] +# * a.flatten #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +# */ +# static VALUE +# rb_ary_flatten(ary) +# VALUE ary; +# { +# ary = rb_obj_dup(ary); +# rb_ary_flatten_bang(ary); +# return ary; +# } +# +# ... +# +# void +# Init_Array() +# { +# ... +# rb_define_method(rb_cArray, "flatten", rb_ary_flatten, 0); +# +# Here RDoc will determine from the rb_define_method line that there's a +# method called "flatten" in class Array, and will look for the implementation +# in the method rb_ary_flatten. It will then use the comment from that +# method in the HTML output. This method must be in the same source file +# as the rb_define_method. +# +# The comment blocks may include special directives: +# +# [Document-class: +name+] +# Documentation for the named class. +# +# [Document-module: +name+] +# Documentation for the named module. +# +# [Document-const: +name+] +# Documentation for the named +rb_define_const+. +# +# Constant values can be supplied on the first line of the comment like so: +# +# /* 300: The highest possible score in bowling */ +# rb_define_const(cFoo, "PERFECT", INT2FIX(300)); +# +# The value can contain internal colons so long as they are escaped with a \ +# +# [Document-global: +name+] +# Documentation for the named +rb_define_global_const+ +# +# [Document-variable: +name+] +# Documentation for the named +rb_define_variable+ +# +# [Document-method: +method_name+] +# Documentation for the named method. Use this when the method name is +# unambiguous. +# +# [Document-method: <tt>ClassName::method_name<tt>] +# Documentation for a singleton method in the given class. Use this when +# the method name alone is ambiguous. +# +# [Document-method: <tt>ClassName#method_name<tt>] +# Documentation for a instance method in the given class. Use this when the +# method name alone is ambiguous. +# +# [Document-attr: +name+] +# Documentation for the named attribute. +# +# [call-seq: <i>text up to an empty line</i>] +# Because C source doesn't give descriptive names to Ruby-level parameters, +# you need to document the calling sequence explicitly +# +# In addition, RDoc assumes by default that the C method implementing a +# Ruby function is in the same source file as the rb_define_method call. +# If this isn't the case, add the comment: +# +# rb_define_method(....); // in filename +# +# As an example, we might have an extension that defines multiple classes +# in its Init_xxx method. We could document them using +# +# /* +# * Document-class: MyClass +# * +# * Encapsulate the writing and reading of the configuration +# * file. ... +# */ +# +# /* +# * Document-method: read_value +# * +# * call-seq: +# * cfg.read_value(key) -> value +# * cfg.read_value(key} { |key| } -> value +# * +# * Return the value corresponding to +key+ from the configuration. +# * In the second form, if the key isn't found, invoke the +# * block and return its value. +# */ + +class RDoc::Parser::C < RDoc::Parser + + parse_files_matching(/\.(?:([CcHh])\1?|c([+xp])\2|y)\z/) + + include RDoc::Text + + ## + # Maps C variable names to names of Ruby classes or modules + + attr_reader :classes + + ## + # C file the parser is parsing + + attr_accessor :content + + ## + # Dependencies from a missing enclosing class to the classes in + # missing_dependencies that depend upon it. + + attr_reader :enclosure_dependencies + + ## + # Maps C variable names to names of Ruby classes (and singleton classes) + + attr_reader :known_classes + + ## + # Classes found while parsing the C file that were not yet registered due to + # a missing enclosing class. These are processed by do_missing + + attr_reader :missing_dependencies + + ## + # Maps C variable names to names of Ruby singleton classes + + attr_reader :singleton_classes + + ## + # The TopLevel items in the parsed file belong to + + attr_reader :top_level + + ## + # Prepares for parsing a C file. See RDoc::Parser#initialize for details on + # the arguments. + + def initialize top_level, file_name, content, options, stats + super + + @known_classes = RDoc::KNOWN_CLASSES.dup + @content = handle_tab_width handle_ifdefs_in @content + @file_dir = File.dirname @file_name + + @classes = load_variable_map :c_class_variables + @singleton_classes = load_variable_map :c_singleton_class_variables + + # class_variable => { function => [method, ...] } + @methods = Hash.new { |h, f| h[f] = Hash.new { |i, m| i[m] = [] } } + + # missing variable => [handle_class_module arguments] + @missing_dependencies = {} + + # missing enclosure variable => [dependent handle_class_module arguments] + @enclosure_dependencies = Hash.new { |h, k| h[k] = [] } + @enclosure_dependencies.instance_variable_set :@missing_dependencies, + @missing_dependencies + + @enclosure_dependencies.extend TSort + + def @enclosure_dependencies.tsort_each_node &block + each_key(&block) + rescue TSort::Cyclic => e + cycle_vars = e.message.scan(/"(.*?)"/).flatten + + cycle = cycle_vars.sort.map do |var_name| + delete var_name + + var_name, type, mod_name, = @missing_dependencies[var_name] + + "#{type} #{mod_name} (#{var_name})" + end.join ', ' + + warn "Unable to create #{cycle} due to a cyclic class or module creation" + + retry + end + + def @enclosure_dependencies.tsort_each_child node, &block + fetch(node, []).each(&block) + end + end + + ## + # Removes duplicate call-seq entries for methods using the same + # implementation. + + def deduplicate_call_seq + @methods.each do |var_name, functions| + class_name = @known_classes[var_name] + class_obj = find_class var_name, class_name + + functions.each_value do |method_names| + next if method_names.length == 1 + + method_names.each do |method_name| + deduplicate_method_name class_obj, method_name + end + end + end + end + + ## + # If two ruby methods share a C implementation (and comment) this + # deduplicates the examples in the call_seq for the method to reduce + # confusion in the output. + + def deduplicate_method_name class_obj, method_name # :nodoc: + return unless + method = class_obj.method_list.find { |m| m.name == method_name } + return unless call_seq = method.call_seq + + method_name = method_name[0, 1] if method_name =~ /\A\[/ + + entries = call_seq.split "\n" + + matching = entries.select do |entry| + entry =~ /^\w*\.?#{Regexp.escape method_name}/ or + entry =~ /\s#{Regexp.escape method_name}\s/ + end + + method.call_seq = matching.join "\n" + end + + ## + # Scans #content for rb_define_alias + + def do_aliases + @content.scan(/rb_define_alias\s*\( + \s*(\w+), + \s*"(.+?)", + \s*"(.+?)" + \s*\)/xm) do |var_name, new_name, old_name| + class_name = @known_classes[var_name] + + unless class_name then + @options.warn "Enclosing class or module %p for alias %s %s is not known" % [ + var_name, new_name, old_name] + next + end + + class_obj = find_class var_name, class_name + + al = RDoc::Alias.new '', old_name, new_name, '' + al.singleton = @singleton_classes.key? var_name + + comment = find_alias_comment var_name, new_name, old_name + + comment.normalize + + al.comment = comment + + al.record_location @top_level + + class_obj.add_alias al + @stats.add_alias al + end + end + + ## + # Scans #content for rb_attr and rb_define_attr + + def do_attrs + @content.scan(/rb_attr\s*\( + \s*(\w+), + \s*([\w"()]+), + \s*([01]), + \s*([01]), + \s*\w+\);/xm) do |var_name, attr_name, read, write| + handle_attr var_name, attr_name, read, write + end + + @content.scan(%r%rb_define_attr\( + \s*([\w\.]+), + \s*"([^"]+)", + \s*(\d+), + \s*(\d+)\s*\); + %xm) do |var_name, attr_name, read, write| + handle_attr var_name, attr_name, read, write + end + end + + ## + # Scans #content for boot_defclass + + def do_boot_defclass + @content.scan(/(\w+)\s*=\s*boot_defclass\s*\(\s*"(\w+?)",\s*(\w+?)\s*\)/) do + |var_name, class_name, parent| + parent = nil if parent == "0" + handle_class_module(var_name, :class, class_name, parent, nil) + end + end + + ## + # Scans #content for rb_define_class, boot_defclass, rb_define_class_under + # and rb_singleton_class + + def do_classes + do_boot_defclass + do_define_class + do_define_class_under + do_singleton_class + do_struct_define_without_accessor + end + + ## + # Scans #content for rb_define_variable, rb_define_readonly_variable, + # rb_define_const and rb_define_global_const + + def do_constants + @content.scan(%r%\Wrb_define_ + ( variable | + readonly_variable | + const | + global_const ) + \s*\( + (?:\s*(\w+),)? + \s*"(\w+)", + \s*(.*?)\s*\)\s*; + %xm) do |type, var_name, const_name, definition| + var_name = "rb_cObject" if !var_name or var_name == "rb_mKernel" + handle_constants type, var_name, const_name, definition + end + + @content.scan(%r% + \Wrb_curses_define_const + \s*\( + \s* + (\w+) + \s* + \) + \s*;%xm) do |consts| + const = consts.first + + handle_constants 'const', 'mCurses', const, "UINT2NUM(#{const})" + end + + @content.scan(%r% + \Wrb_file_const + \s*\( + \s* + "([^"]+)", + \s* + (.*?) + \s* + \) + \s*;%xm) do |name, value| + handle_constants 'const', 'rb_mFConst', name, value + end + end + + ## + # Scans #content for rb_define_class + + def do_define_class + # The '.' lets us handle SWIG-generated files + @content.scan(/([\w\.]+)\s* = \s*rb_define_class\s* + \( + \s*"(\w+)", + \s*(\w+)\s* + \)/mx) do |var_name, class_name, parent| + handle_class_module(var_name, :class, class_name, parent, nil) + end + end + + ## + # Scans #content for rb_define_class_under + + def do_define_class_under + @content.scan(/([\w\.]+)\s* = # var_name + \s*rb_define_class_under\s* + \( + \s* (\w+), # under + \s* "(\w+)", # class_name + \s* + (?: + ([\w\*\s\(\)\.\->]+) | # parent_name + rb_path2class\("([\w:]+)"\) # path + ) + \s* + \) + /mx) do |var_name, under, class_name, parent_name, path| + parent = path || parent_name + + handle_class_module var_name, :class, class_name, parent, under + end + end + + ## + # Scans #content for rb_define_module + + def do_define_module + @content.scan(/(\w+)\s* = \s*rb_define_module\s*\(\s*"(\w+)"\s*\)/mx) do + |var_name, class_name| + handle_class_module(var_name, :module, class_name, nil, nil) + end + end + + ## + # Scans #content for rb_define_module_under + + def do_define_module_under + @content.scan(/(\w+)\s* = \s*rb_define_module_under\s* + \( + \s*(\w+), + \s*"(\w+)" + \s*\)/mx) do |var_name, in_module, class_name| + handle_class_module(var_name, :module, class_name, nil, in_module) + end + end + + ## + # Scans #content for rb_include_module + + def do_includes + @content.scan(/rb_include_module\s*\(\s*(\w+?),\s*(\w+?)\s*\)/) do |c,m| + next unless cls = @classes[c] + m = @known_classes[m] || m + + comment = RDoc::Comment.new '', @top_level + incl = cls.add_include RDoc::Include.new(m, comment) + incl.record_location @top_level + end + end + + ## + # Scans #content for rb_define_method, rb_define_singleton_method, + # rb_define_module_function, rb_define_private_method, + # rb_define_global_function and define_filetest_function + + def do_methods + @content.scan(%r%rb_define_ + ( + singleton_method | + method | + module_function | + private_method + ) + \s*\(\s*([\w\.]+), + \s*"([^"]+)", + \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\(|\(METHOD\))?(\w+)\)?, + \s*(-?\w+)\s*\) + (?:;\s*/[*/]\s+in\s+(\w+?\.(?:cpp|c|y)))? + %xm) do |type, var_name, meth_name, function, param_count, source_file| + + # Ignore top-object and weird struct.c dynamic stuff + next if var_name == "ruby_top_self" + next if var_name == "nstr" + + var_name = "rb_cObject" if var_name == "rb_mKernel" + handle_method(type, var_name, meth_name, function, param_count, + source_file) + end + + @content.scan(%r%rb_define_global_function\s*\( + \s*"([^"]+)", + \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?, + \s*(-?\w+)\s*\) + (?:;\s*/[*/]\s+in\s+(\w+?\.[cy]))? + %xm) do |meth_name, function, param_count, source_file| + handle_method("method", "rb_mKernel", meth_name, function, param_count, + source_file) + end + + @content.scan(/define_filetest_function\s*\( + \s*"([^"]+)", + \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?, + \s*(-?\w+)\s*\)/xm) do |meth_name, function, param_count| + + handle_method("method", "rb_mFileTest", meth_name, function, param_count) + handle_method("singleton_method", "rb_cFile", meth_name, function, + param_count) + end + end + + ## + # Creates classes and module that were missing were defined due to the file + # order being different than the declaration order. + + def do_missing + return if @missing_dependencies.empty? + + @enclosure_dependencies.tsort.each do |in_module| + arguments = @missing_dependencies.delete in_module + + next unless arguments # dependency on existing class + + handle_class_module(*arguments) + end + end + + ## + # Scans #content for rb_define_module and rb_define_module_under + + def do_modules + do_define_module + do_define_module_under + end + + ## + # Scans #content for rb_singleton_class + + def do_singleton_class + @content.scan(/([\w\.]+)\s* = \s*rb_singleton_class\s* + \( + \s*(\w+) + \s*\)/mx) do |sclass_var, class_var| + handle_singleton sclass_var, class_var + end + end + + ## + # Scans #content for struct_define_without_accessor + + def do_struct_define_without_accessor + @content.scan(/([\w\.]+)\s* = \s*rb_struct_define_without_accessor\s* + \( + \s*"(\w+)", # Class name + \s*(\w+), # Parent class + \s*\w+, # Allocation function + (\s*"\w+",)* # Attributes + \s*NULL + \)/mx) do |var_name, class_name, parent| + handle_class_module(var_name, :class, class_name, parent, nil) + end + end + + ## + # Finds the comment for an alias on +class_name+ from +new_name+ to + # +old_name+ + + def find_alias_comment class_name, new_name, old_name + content =~ %r%((?>/\*.*?\*/\s+)) + rb_define_alias\(\s*#{Regexp.escape class_name}\s*, + \s*"#{Regexp.escape new_name}"\s*, + \s*"#{Regexp.escape old_name}"\s*\);%xm + + RDoc::Comment.new($1 || '', @top_level) + end + + ## + # Finds a comment for rb_define_attr, rb_attr or Document-attr. + # + # +var_name+ is the C class variable the attribute is defined on. + # +attr_name+ is the attribute's name. + # + # +read+ and +write+ are the read/write flags ('1' or '0'). Either both or + # neither must be provided. + + def find_attr_comment var_name, attr_name, read = nil, write = nil + attr_name = Regexp.escape attr_name + + rw = if read and write then + /\s*#{read}\s*,\s*#{write}\s*/xm + else + /.*?/m + end + + comment = if @content =~ %r%((?>/\*.*?\*/\s+)) + rb_define_attr\((?:\s*#{var_name},)?\s* + "#{attr_name}"\s*, + #{rw}\)\s*;%xm then + $1 + elsif @content =~ %r%((?>/\*.*?\*/\s+)) + rb_attr\(\s*#{var_name}\s*, + \s*#{attr_name}\s*, + #{rw},.*?\)\s*;%xm then + $1 + elsif @content =~ %r%(/\*.*?(?:\s*\*\s*)?) + Document-attr:\s#{attr_name}\s*?\n + ((?>(.|\n)*?\*/))%x then + "#{$1}\n#{$2}" + else + '' + end + + RDoc::Comment.new comment, @top_level + end + + ## + # Find the C code corresponding to a Ruby method + + def find_body class_name, meth_name, meth_obj, file_content, quiet = false + case file_content + when %r%((?>/\*.*?\*/\s*)?) + ((?:(?:\w+)\s+)? + (?:intern\s+)?VALUE\s+#{meth_name} + \s*(\([^)]*\))([^;]|$))%xm then + comment = RDoc::Comment.new $1, @top_level + body = $2 + offset, = $~.offset(2) + + comment.remove_private if comment + + # try to find the whole body + body = $& if /#{Regexp.escape body}[^(]*?\{.*?^\}/m =~ file_content + + # The comment block may have been overridden with a 'Document-method' + # block. This happens in the interpreter when multiple methods are + # vectored through to the same C method but those methods are logically + # distinct (for example Kernel.hash and Kernel.object_id share the same + # implementation + + override_comment = find_override_comment class_name, meth_obj + comment = override_comment if override_comment + + comment.normalize + find_modifiers comment, meth_obj if comment + + #meth_obj.params = params + meth_obj.start_collecting_tokens + tk = RDoc::RubyToken::Token.new nil, 1, 1 + tk.set_text body + meth_obj.add_token tk + meth_obj.comment = comment + meth_obj.offset = offset + meth_obj.line = file_content[0, offset].count("\n") + 1 + + body + when %r%((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+#{meth_name}\s+(\w+))%m then + comment = RDoc::Comment.new $1, @top_level + body = $2 + offset = $~.offset(2).first + + find_body class_name, $3, meth_obj, file_content, true + + comment.normalize + find_modifiers comment, meth_obj + + meth_obj.start_collecting_tokens + tk = RDoc::RubyToken::Token.new nil, 1, 1 + tk.set_text body + meth_obj.add_token tk + meth_obj.comment = comment + meth_obj.offset = offset + meth_obj.line = file_content[0, offset].count("\n") + 1 + + body + when %r%^\s*\#\s*define\s+#{meth_name}\s+(\w+)%m then + # with no comment we hope the aliased definition has it and use it's + # definition + + body = find_body(class_name, $1, meth_obj, file_content, true) + + return body if body + + @options.warn "No definition for #{meth_name}" + false + else # No body, but might still have an override comment + comment = find_override_comment class_name, meth_obj + + if comment then + comment.normalize + find_modifiers comment, meth_obj + meth_obj.comment = comment + + '' + else + @options.warn "No definition for #{meth_name}" + false + end + end + end + + ## + # Finds a RDoc::NormalClass or RDoc::NormalModule for +raw_name+ + + def find_class(raw_name, name) + unless @classes[raw_name] + if raw_name =~ /^rb_m/ + container = @top_level.add_module RDoc::NormalModule, name + else + container = @top_level.add_class RDoc::NormalClass, name + end + + container.record_location @top_level + @classes[raw_name] = container + end + @classes[raw_name] + end + + ## + # Look for class or module documentation above Init_+class_name+(void), + # in a Document-class +class_name+ (or module) comment or above an + # rb_define_class (or module). If a comment is supplied above a matching + # Init_ and a rb_define_class the Init_ comment is used. + # + # /* + # * This is a comment for Foo + # */ + # Init_Foo(void) { + # VALUE cFoo = rb_define_class("Foo", rb_cObject); + # } + # + # /* + # * Document-class: Foo + # * This is a comment for Foo + # */ + # Init_foo(void) { + # VALUE cFoo = rb_define_class("Foo", rb_cObject); + # } + # + # /* + # * This is a comment for Foo + # */ + # VALUE cFoo = rb_define_class("Foo", rb_cObject); + + def find_class_comment class_name, class_mod + comment = nil + + if @content =~ %r% + ((?>/\*.*?\*/\s+)) + (static\s+)? + void\s+ + Init_#{class_name}\s*(?:_\(\s*)?\(\s*(?:void\s*)?\)%xmi then + comment = $1.sub(%r%Document-(?:class|module):\s+#{class_name}%, '') + elsif @content =~ %r%Document-(?:class|module):\s+#{class_name}\s*? + (?:<\s+[:,\w]+)?\n((?>.*?\*/))%xm then + comment = "/*\n#{$1}" + elsif @content =~ %r%((?>/\*.*?\*/\s+)) + ([\w\.\s]+\s* = \s+)?rb_define_(class|module)[\t (]*?"(#{class_name})"%xm then + comment = $1 + elsif @content =~ %r%((?>/\*.*?\*/\s+)) + ([\w\. \t]+ = \s+)?rb_define_(class|module)_under[\t\w, (]*?"(#{class_name.split('::').last})"%xm then + comment = $1 + else + comment = '' + end + + comment = RDoc::Comment.new comment, @top_level + comment.normalize + + look_for_directives_in class_mod, comment + + class_mod.add_comment comment, @top_level + end + + ## + # Finds a comment matching +type+ and +const_name+ either above the + # comment or in the matching Document- section. + + def find_const_comment(type, const_name, class_name = nil) + comment = if @content =~ %r%((?>^\s*/\*.*?\*/\s+)) + rb_define_#{type}\((?:\s*(\w+),)?\s* + "#{const_name}"\s*, + .*?\)\s*;%xmi then + $1 + elsif class_name and + @content =~ %r%Document-(?:const|global|variable):\s + #{class_name}::#{const_name} + \s*?\n((?>.*?\*/))%xm then + "/*\n#{$1}" + elsif @content =~ %r%Document-(?:const|global|variable): + \s#{const_name} + \s*?\n((?>.*?\*/))%xm then + "/*\n#{$1}" + else + '' + end + + RDoc::Comment.new comment, @top_level + end + + ## + # Handles modifiers in +comment+ and updates +meth_obj+ as appropriate. + + def find_modifiers comment, meth_obj + comment.normalize + comment.extract_call_seq meth_obj + + look_for_directives_in meth_obj, comment + end + + ## + # Finds a <tt>Document-method</tt> override for +meth_obj+ on +class_name+ + + def find_override_comment class_name, meth_obj + name = Regexp.escape meth_obj.name + prefix = Regexp.escape meth_obj.name_prefix + + comment = if @content =~ %r%Document-method: + \s+#{class_name}#{prefix}#{name} + \s*?\n((?>.*?\*/))%xm then + "/*#{$1}" + elsif @content =~ %r%Document-method: + \s#{name}\s*?\n((?>.*?\*/))%xm then + "/*#{$1}" + end + + return unless comment + + RDoc::Comment.new comment, @top_level + end + + ## + # Creates a new RDoc::Attr +attr_name+ on class +var_name+ that is either + # +read+, +write+ or both + + def handle_attr(var_name, attr_name, read, write) + rw = '' + rw << 'R' if '1' == read + rw << 'W' if '1' == write + + class_name = @known_classes[var_name] + + return unless class_name + + class_obj = find_class var_name, class_name + + return unless class_obj + + comment = find_attr_comment var_name, attr_name + comment.normalize + + name = attr_name.gsub(/rb_intern\("([^"]+)"\)/, '\1') + + attr = RDoc::Attr.new '', name, rw, comment + + attr.record_location @top_level + class_obj.add_attribute attr + @stats.add_attribute attr + end + + ## + # Creates a new RDoc::NormalClass or RDoc::NormalModule based on +type+ + # named +class_name+ in +parent+ which was assigned to the C +var_name+. + + def handle_class_module(var_name, type, class_name, parent, in_module) + parent_name = @known_classes[parent] || parent + + if in_module then + enclosure = @classes[in_module] || @store.find_c_enclosure(in_module) + + if enclosure.nil? and enclosure = @known_classes[in_module] then + enc_type = /^rb_m/ =~ in_module ? :module : :class + handle_class_module in_module, enc_type, enclosure, nil, nil + enclosure = @classes[in_module] + end + + unless enclosure then + @enclosure_dependencies[in_module] << var_name + @missing_dependencies[var_name] = + [var_name, type, class_name, parent, in_module] + + return + end + else + enclosure = @top_level + end + + if type == :class then + full_name = if RDoc::ClassModule === enclosure then + enclosure.full_name + "::#{class_name}" + else + class_name + end + + if @content =~ %r%Document-class:\s+#{full_name}\s*<\s+([:,\w]+)% then + parent_name = $1 + end + + cm = enclosure.add_class RDoc::NormalClass, class_name, parent_name + else + cm = enclosure.add_module RDoc::NormalModule, class_name + end + + cm.record_location enclosure.top_level + + find_class_comment cm.full_name, cm + + case cm + when RDoc::NormalClass + @stats.add_class cm + when RDoc::NormalModule + @stats.add_module cm + end + + @classes[var_name] = cm + @known_classes[var_name] = cm.full_name + @store.add_c_enclosure var_name, cm + end + + ## + # Adds constants. By providing some_value: at the start of the comment you + # can override the C value of the comment to give a friendly definition. + # + # /* 300: The perfect score in bowling */ + # rb_define_const(cFoo, "PERFECT", INT2FIX(300); + # + # Will override <tt>INT2FIX(300)</tt> with the value +300+ in the output + # RDoc. Values may include quotes and escaped colons (\:). + + def handle_constants(type, var_name, const_name, definition) + class_name = @known_classes[var_name] + + return unless class_name + + class_obj = find_class var_name, class_name + + unless class_obj then + @options.warn 'Enclosing class or module %p is not known' % [const_name] + return + end + + comment = find_const_comment type, const_name, class_name + comment.normalize + + # In the case of rb_define_const, the definition and comment are in + # "/* definition: comment */" form. The literal ':' and '\' characters + # can be escaped with a backslash. + if type.downcase == 'const' then + no_match, new_definition, new_comment = comment.text.split(/(\A.*):/) + + if no_match and no_match.empty? then + if new_definition.empty? then # Default to literal C definition + new_definition = definition + else + new_definition.gsub!("\:", ":") + new_definition.gsub!("\\", '\\') + end + + new_definition.sub!(/\A(\s+)/, '') + + new_comment = "#{$1}#{new_comment.lstrip}" + + new_comment = RDoc::Comment.new new_comment, @top_level + + con = RDoc::Constant.new const_name, new_definition, new_comment + else + con = RDoc::Constant.new const_name, definition, comment + end + else + con = RDoc::Constant.new const_name, definition, comment + end + + con.record_location @top_level + @stats.add_constant con + class_obj.add_constant con + end + + ## + # Removes #ifdefs that would otherwise confuse us + + def handle_ifdefs_in(body) + body.gsub(/^#ifdef HAVE_PROTOTYPES.*?#else.*?\n(.*?)#endif.*?\n/m, '\1') + end + + ## + # Adds an RDoc::AnyMethod +meth_name+ defined on a class or module assigned + # to +var_name+. +type+ is the type of method definition function used. + # +singleton_method+ and +module_function+ create a singleton method. + + def handle_method(type, var_name, meth_name, function, param_count, + source_file = nil) + class_name = @known_classes[var_name] + singleton = @singleton_classes.key? var_name + + @methods[var_name][function] << meth_name + + return unless class_name + + class_obj = find_class var_name, class_name + + if class_obj then + if meth_name == 'initialize' then + meth_name = 'new' + singleton = true + type = 'method' # force public + end + + meth_obj = RDoc::AnyMethod.new '', meth_name + meth_obj.c_function = function + meth_obj.singleton = + singleton || %w[singleton_method module_function].include?(type) + + p_count = Integer(param_count) rescue -1 + + if source_file then + file_name = File.join @file_dir, source_file + + if File.exist? file_name then + file_content = File.read file_name + else + @options.warn "unknown source #{source_file} for #{meth_name} in #{@file_name}" + end + else + file_content = @content + end + + body = find_body class_name, function, meth_obj, file_content + + if body and meth_obj.document_self then + meth_obj.params = if p_count < -1 then # -2 is Array + '(*args)' + elsif p_count == -1 then # argc, argv + rb_scan_args body + else + "(#{(1..p_count).map { |i| "p#{i}" }.join ', '})" + end + + + meth_obj.record_location @top_level + class_obj.add_method meth_obj + @stats.add_method meth_obj + meth_obj.visibility = :private if 'private_method' == type + end + end + end + + ## + # Registers a singleton class +sclass_var+ as a singleton of +class_var+ + + def handle_singleton sclass_var, class_var + class_name = @known_classes[class_var] + + @known_classes[sclass_var] = class_name + @singleton_classes[sclass_var] = class_name + end + + ## + # Normalizes tabs in +body+ + + def handle_tab_width(body) + if /\t/ =~ body + tab_width = @options.tab_width + body.split(/\n/).map do |line| + 1 while line.gsub!(/\t+/) do + ' ' * (tab_width * $&.length - $`.length % tab_width) + end && $~ + line + end.join "\n" + else + body + end + end + + ## + # Loads the variable map with the given +name+ from the RDoc::Store, if + # present. + + def load_variable_map map_name + return {} unless files = @store.cache[map_name] + return {} unless name_map = files[@file_name] + + class_map = {} + + name_map.each do |variable, name| + next unless mod = @store.find_class_or_module(name) + + class_map[variable] = if map_name == :c_class_variables then + mod + else + name + end + @known_classes[variable] = name + end + + class_map + end + + ## + # Look for directives in a normal comment block: + # + # /* + # * :title: My Awesome Project + # */ + # + # This method modifies the +comment+ + + def look_for_directives_in context, comment + @preprocess.handle comment, context do |directive, param| + case directive + when 'main' then + @options.main_page = param + '' + when 'title' then + @options.default_title = param if @options.respond_to? :default_title= + '' + end + end + + comment + end + + ## + # Extracts parameters from the +method_body+ and returns a method + # parameter string. Follows 1.9.3dev's scan-arg-spec, see README.EXT + + def rb_scan_args method_body + method_body =~ /rb_scan_args\((.*?)\)/m + return '(*args)' unless $1 + + $1.split(/,/)[2] =~ /"(.*?)"/ # format argument + format = $1.split(//) + + lead = opt = trail = 0 + + if format.first =~ /\d/ then + lead = $&.to_i + format.shift + if format.first =~ /\d/ then + opt = $&.to_i + format.shift + if format.first =~ /\d/ then + trail = $&.to_i + format.shift + block_arg = true + end + end + end + + if format.first == '*' and not block_arg then + var = true + format.shift + if format.first =~ /\d/ then + trail = $&.to_i + format.shift + end + end + + if format.first == ':' then + hash = true + format.shift + end + + if format.first == '&' then + block = true + format.shift + end + + # if the format string is not empty there's a bug in the C code, ignore it + + args = [] + position = 1 + + (1...(position + lead)).each do |index| + args << "p#{index}" + end + + position += lead + + (position...(position + opt)).each do |index| + args << "p#{index} = v#{index}" + end + + position += opt + + if var then + args << '*args' + position += 1 + end + + (position...(position + trail)).each do |index| + args << "p#{index}" + end + + position += trail + + if hash then + args << "p#{position} = {}" + end + + args << '&block' if block + + "(#{args.join ', '})" + end + + ## + # Removes lines that are commented out that might otherwise get picked up + # when scanning for classes and methods + + def remove_commented_out_lines + @content.gsub!(%r%//.*rb_define_%, '//') + end + + ## + # Extracts the classes, modules, methods, attributes, constants and aliases + # from a C file and returns an RDoc::TopLevel for this file + + def scan + remove_commented_out_lines + + do_modules + do_classes + do_missing + + do_constants + do_methods + do_includes + do_aliases + do_attrs + + deduplicate_call_seq + + @store.add_c_variables self + + @top_level + end + +end + diff --git a/jni/ruby/lib/rdoc/parser/changelog.rb b/jni/ruby/lib/rdoc/parser/changelog.rb new file mode 100644 index 0000000..a3567c1 --- /dev/null +++ b/jni/ruby/lib/rdoc/parser/changelog.rb @@ -0,0 +1,198 @@ +require 'time' + +## +# A ChangeLog file parser. +# +# This parser converts a ChangeLog into an RDoc::Markup::Document. When +# viewed as HTML a ChangeLog page will have an entry for each day's entries in +# the sidebar table of contents. +# +# This parser is meant to parse the MRI ChangeLog, but can be used to parse any +# {GNU style Change +# Log}[http://www.gnu.org/prep/standards/html_node/Style-of-Change-Logs.html]. + +class RDoc::Parser::ChangeLog < RDoc::Parser + + include RDoc::Parser::Text + + parse_files_matching(/(\/|\\|\A)ChangeLog[^\/\\]*\z/) + + ## + # Attaches the +continuation+ of the previous line to the +entry_body+. + # + # Continued function listings are joined together as a single entry. + # Continued descriptions are joined to make a single paragraph. + + def continue_entry_body entry_body, continuation + return unless last = entry_body.last + + if last =~ /\)\s*\z/ and continuation =~ /\A\(/ then + last.sub!(/\)\s*\z/, ',') + continuation.sub!(/\A\(/, '') + end + + if last =~ /\s\z/ then + last << continuation + else + last << ' ' << continuation + end + end + + ## + # Creates an RDoc::Markup::Document given the +groups+ of ChangeLog entries. + + def create_document groups + doc = RDoc::Markup::Document.new + doc.omit_headings_below = 2 + doc.file = @top_level + + doc << RDoc::Markup::Heading.new(1, File.basename(@file_name)) + doc << RDoc::Markup::BlankLine.new + + groups.sort_by do |day,| day end.reverse_each do |day, entries| + doc << RDoc::Markup::Heading.new(2, day.dup) + doc << RDoc::Markup::BlankLine.new + + doc.concat create_entries entries + end + + doc + end + + ## + # Returns a list of ChangeLog entries an RDoc::Markup nodes for the given + # +entries+. + + def create_entries entries + out = [] + + entries.each do |entry, items| + out << RDoc::Markup::Heading.new(3, entry) + out << RDoc::Markup::BlankLine.new + + out << create_items(items) + end + + out + end + + ## + # Returns an RDoc::Markup::List containing the given +items+ in the + # ChangeLog + + def create_items items + list = RDoc::Markup::List.new :NOTE + + items.each do |item| + item =~ /\A(.*?(?:\([^)]+\))?):\s*/ + + title = $1 + body = $' + + paragraph = RDoc::Markup::Paragraph.new body + list_item = RDoc::Markup::ListItem.new title, paragraph + list << list_item + end + + list + end + + ## + # Groups +entries+ by date. + + def group_entries entries + entries.group_by do |title, _| + begin + Time.parse(title).strftime '%Y-%m-%d' + rescue NoMethodError, ArgumentError + time, = title.split ' ', 2 + Time.parse(time).strftime '%Y-%m-%d' + end + end + end + + ## + # Parses the entries in the ChangeLog. + # + # Returns an Array of each ChangeLog entry in order of parsing. + # + # A ChangeLog entry is an Array containing the ChangeLog title (date and + # committer) and an Array of ChangeLog items (file and function changed with + # description). + # + # An example result would be: + # + # [ 'Tue Dec 4 08:33:46 2012 Eric Hodel <drbrain@segment7.net>', + # [ 'README.EXT: Converted to RDoc format', + # 'README.EXT.ja: ditto']] + + def parse_entries + entries = [] + entry_name = nil + entry_body = [] + + @content.each_line do |line| + case line + when /^\s*$/ then + next + when /^\w.*/ then + entries << [entry_name, entry_body] if entry_name + + entry_name = $& + + begin + time = Time.parse entry_name + # HACK Ruby 1.8 does not raise ArgumentError for Time.parse "Other" + entry_name = nil unless entry_name =~ /#{time.year}/ + rescue NoMethodError + # HACK Ruby 2.1.2 and earlier raises NoMethodError if time part is absent + entry_name.split ' ', 2 + rescue ArgumentError + if /out of range/ =~ $!.message + Time.parse(entry_name.split(' ', 2)[0]) rescue entry_name = nil + else + entry_name = nil + end + end + + entry_body = [] + when /^(\t| {8})?\*\s*(.*)/ then # "\t* file.c (func): ..." + entry_body << $2 + when /^(\t| {8})?\s*(\(.*)/ then # "\t(func): ..." + entry = $2 + + if entry_body.last =~ /:/ then + entry_body << entry + else + continue_entry_body entry_body, entry + end + when /^(\t| {8})?\s*(.*)/ then + continue_entry_body entry_body, $2 + end + end + + entries << [entry_name, entry_body] if entry_name + + entries.reject! do |(entry,_)| + entry == nil + end + + entries + end + + ## + # Converts the ChangeLog into an RDoc::Markup::Document + + def scan + entries = parse_entries + grouped_entries = group_entries entries + + doc = create_document grouped_entries + + @top_level.comment = doc + + @top_level + end + +end + diff --git a/jni/ruby/lib/rdoc/parser/markdown.rb b/jni/ruby/lib/rdoc/parser/markdown.rb new file mode 100644 index 0000000..6fd88cf --- /dev/null +++ b/jni/ruby/lib/rdoc/parser/markdown.rb @@ -0,0 +1,23 @@ +## +# Parse a Markdown format file. The parsed RDoc::Markup::Document is attached +# as a file comment. + +class RDoc::Parser::Markdown < RDoc::Parser + + include RDoc::Parser::Text + + parse_files_matching(/\.(md|markdown)(?:\.[^.]+)?$/) + + ## + # Creates an Markdown-format TopLevel for the given file. + + def scan + comment = RDoc::Comment.new @content, @top_level + comment.format = 'markdown' + + @top_level.comment = comment + end + +end + + diff --git a/jni/ruby/lib/rdoc/parser/rd.rb b/jni/ruby/lib/rdoc/parser/rd.rb new file mode 100644 index 0000000..09069ae --- /dev/null +++ b/jni/ruby/lib/rdoc/parser/rd.rb @@ -0,0 +1,22 @@ +## +# Parse a RD format file. The parsed RDoc::Markup::Document is attached as a +# file comment. + +class RDoc::Parser::RD < RDoc::Parser + + include RDoc::Parser::Text + + parse_files_matching(/\.rd(?:\.[^.]+)?$/) + + ## + # Creates an rd-format TopLevel for the given file. + + def scan + comment = RDoc::Comment.new @content, @top_level + comment.format = 'rd' + + @top_level.comment = comment + end + +end + diff --git a/jni/ruby/lib/rdoc/parser/ruby.rb b/jni/ruby/lib/rdoc/parser/ruby.rb new file mode 100644 index 0000000..ce1083e --- /dev/null +++ b/jni/ruby/lib/rdoc/parser/ruby.rb @@ -0,0 +1,2160 @@ +## +# This file contains stuff stolen outright from: +# +# rtags.rb - +# ruby-lex.rb - ruby lexcal analyzer +# ruby-token.rb - ruby tokens +# by Keiju ISHITSUKA (Nippon Rational Inc.) +# + +$TOKEN_DEBUG ||= nil + +## +# Extracts code elements from a source file returning a TopLevel object +# containing the constituent file elements. +# +# This file is based on rtags +# +# RubyParser understands how to document: +# * classes +# * modules +# * methods +# * constants +# * aliases +# * private, public, protected +# * private_class_function, public_class_function +# * module_function +# * attr, attr_reader, attr_writer, attr_accessor +# * extra accessors given on the command line +# * metaprogrammed methods +# * require +# * include +# +# == Method Arguments +# +#-- +# NOTE: I don't think this works, needs tests, remove the paragraph following +# this block when known to work +# +# The parser extracts the arguments from the method definition. You can +# override this with a custom argument definition using the :args: directive: +# +# ## +# # This method tries over and over until it is tired +# +# def go_go_go(thing_to_try, tries = 10) # :args: thing_to_try +# puts thing_to_try +# go_go_go thing_to_try, tries - 1 +# end +# +# If you have a more-complex set of overrides you can use the :call-seq: +# directive: +#++ +# +# The parser extracts the arguments from the method definition. You can +# override this with a custom argument definition using the :call-seq: +# directive: +# +# ## +# # This method can be called with a range or an offset and length +# # +# # :call-seq: +# # my_method(Range) +# # my_method(offset, length) +# +# def my_method(*args) +# end +# +# The parser extracts +yield+ expressions from method bodies to gather the +# yielded argument names. If your method manually calls a block instead of +# yielding or you want to override the discovered argument names use +# the :yields: directive: +# +# ## +# # My method is awesome +# +# def my_method(&block) # :yields: happy, times +# block.call 1, 2 +# end +# +# == Metaprogrammed Methods +# +# To pick up a metaprogrammed method, the parser looks for a comment starting +# with '##' before an identifier: +# +# ## +# # This is a meta-programmed method! +# +# add_my_method :meta_method, :arg1, :arg2 +# +# The parser looks at the token after the identifier to determine the name, in +# this example, :meta_method. If a name cannot be found, a warning is printed +# and 'unknown is used. +# +# You can force the name of a method using the :method: directive: +# +# ## +# # :method: some_method! +# +# By default, meta-methods are instance methods. To indicate that a method is +# a singleton method instead use the :singleton-method: directive: +# +# ## +# # :singleton-method: +# +# You can also use the :singleton-method: directive with a name: +# +# ## +# # :singleton-method: some_method! +# +# You can define arguments for metaprogrammed methods via either the +# :call-seq:, :arg: or :args: directives. +# +# Additionally you can mark a method as an attribute by +# using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:. Just like +# for :method:, the name is optional. +# +# ## +# # :attr_reader: my_attr_name +# +# == Hidden methods and attributes +# +# You can provide documentation for methods that don't appear using +# the :method:, :singleton-method: and :attr: directives: +# +# ## +# # :attr_writer: ghost_writer +# # There is an attribute here, but you can't see it! +# +# ## +# # :method: ghost_method +# # There is a method here, but you can't see it! +# +# ## +# # this is a comment for a regular method +# +# def regular_method() end +# +# Note that by default, the :method: directive will be ignored if there is a +# standard rdocable item following it. + +class RDoc::Parser::Ruby < RDoc::Parser + + parse_files_matching(/\.rbw?$/) + + include RDoc::RubyToken + include RDoc::TokenStream + include RDoc::Parser::RubyTools + + ## + # RDoc::NormalClass type + + NORMAL = "::" + + ## + # RDoc::SingleClass type + + SINGLE = "<<" + + ## + # Creates a new Ruby parser. + + def initialize(top_level, file_name, content, options, stats) + super + + @size = 0 + @token_listeners = nil + @scanner = RDoc::RubyLex.new content, @options + @scanner.exception_on_syntax_error = false + @prev_seek = nil + @markup = @options.markup + @track_visibility = :nodoc != @options.visibility + + @encoding = nil + @encoding = @options.encoding if Object.const_defined? :Encoding + + reset + end + + ## + # Retrieves the read token stream and replaces +pattern+ with +replacement+ + # using gsub. If the result is only a ";" returns an empty string. + + def get_tkread_clean pattern, replacement # :nodoc: + read = get_tkread.gsub(pattern, replacement).strip + return '' if read == ';' + read + end + + ## + # Extracts the visibility information for the visibility token +tk+ + # and +single+ class type identifier. + # + # Returns the visibility type (a string), the visibility (a symbol) and + # +singleton+ if the methods following should be converted to singleton + # methods. + + def get_visibility_information tk, single # :nodoc: + vis_type = tk.name + singleton = single == SINGLE + + vis = + case vis_type + when 'private' then :private + when 'protected' then :protected + when 'public' then :public + when 'private_class_method' then + singleton = true + :private + when 'public_class_method' then + singleton = true + :public + when 'module_function' then + singleton = true + :public + else + raise RDoc::Error, "Invalid visibility: #{tk.name}" + end + + return vis_type, vis, singleton + end + + ## + # Look for the first comment in a file that isn't a shebang line. + + def collect_first_comment + skip_tkspace + comment = '' + comment.force_encoding @encoding if @encoding + first_line = true + first_comment_tk_class = nil + + tk = get_tk + + while TkCOMMENT === tk + if first_line and tk.text =~ /\A#!/ then + skip_tkspace + tk = get_tk + elsif first_line and tk.text =~ /\A#\s*-\*-/ then + first_line = false + skip_tkspace + tk = get_tk + else + break if first_comment_tk_class and not first_comment_tk_class === tk + first_comment_tk_class = tk.class + + first_line = false + comment << tk.text << "\n" + tk = get_tk + + if TkNL === tk then + skip_tkspace false + tk = get_tk + end + end + end + + unget_tk tk + + new_comment comment + end + + ## + # Consumes trailing whitespace from the token stream + + def consume_trailing_spaces # :nodoc: + get_tkread + skip_tkspace false + end + + ## + # Creates a new attribute in +container+ with +name+. + + def create_attr container, single, name, rw, comment # :nodoc: + att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE + record_location att + + container.add_attribute att + @stats.add_attribute att + + att + end + + ## + # Creates a module alias in +container+ at +rhs_name+ (or at the top-level + # for "::") with the name from +constant+. + + def create_module_alias container, constant, rhs_name # :nodoc: + mod = if rhs_name =~ /^::/ then + @store.find_class_or_module rhs_name + else + container.find_module_named rhs_name + end + + container.add_module_alias mod, constant.name, @top_level if mod + end + + ## + # Aborts with +msg+ + + def error(msg) + msg = make_message msg + + abort msg + end + + ## + # Looks for a true or false token. Returns false if TkFALSE or TkNIL are + # found. + + def get_bool + skip_tkspace + tk = get_tk + case tk + when TkTRUE + true + when TkFALSE, TkNIL + false + else + unget_tk tk + true + end + end + + ## + # Look for the name of a class of module (optionally with a leading :: or + # with :: separated named) and return the ultimate name, the associated + # container, and the given name (with the ::). + + def get_class_or_module container, ignore_constants = false + skip_tkspace + name_t = get_tk + given_name = '' + + # class ::A -> A is in the top level + case name_t + when TkCOLON2, TkCOLON3 then # bug + name_t = get_tk + container = @top_level + given_name << '::' + end + + skip_tkspace false + given_name << name_t.name + + while TkCOLON2 === peek_tk do + prev_container = container + container = container.find_module_named name_t.name + container ||= + if ignore_constants then + RDoc::Context.new + else + c = prev_container.add_module RDoc::NormalModule, name_t.name + c.ignore unless prev_container.document_children + @top_level.add_to_classes_or_modules c + c + end + + record_location container + + get_tk + skip_tkspace false + name_t = get_tk + given_name << '::' << name_t.name + end + + skip_tkspace false + + return [container, name_t, given_name] + end + + ## + # Return a superclass, which can be either a constant of an expression + + def get_class_specification + case peek_tk + when TkSELF then return 'self' + when TkGVAR then return '' + end + + res = get_constant + + skip_tkspace false + + get_tkread # empty out read buffer + + tk = get_tk + + case tk + when TkNL, TkCOMMENT, TkSEMICOLON then + unget_tk(tk) + return res + end + + res += parse_call_parameters(tk) + res + end + + ## + # Parse a constant, which might be qualified by one or more class or module + # names + + def get_constant + res = "" + skip_tkspace false + tk = get_tk + + while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do + res += tk.name + tk = get_tk + end + + unget_tk(tk) + res + end + + ## + # Get a constant that may be surrounded by parens + + def get_constant_with_optional_parens + skip_tkspace false + + nest = 0 + + while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do + get_tk + skip_tkspace + nest += 1 + end + + name = get_constant + + while nest > 0 + skip_tkspace + tk = get_tk + nest -= 1 if TkRPAREN === tk + end + + name + end + + ## + # Little hack going on here. In the statement: + # + # f = 2*(1+yield) + # + # We see the RPAREN as the next token, so we need to exit early. This still + # won't catch all cases (such as "a = yield + 1" + + def get_end_token tk # :nodoc: + case tk + when TkLPAREN, TkfLPAREN + TkRPAREN + when TkRPAREN + nil + else + TkNL + end + end + + ## + # Retrieves the method container for a singleton method. + + def get_method_container container, name_t # :nodoc: + prev_container = container + container = container.find_module_named(name_t.name) + + unless container then + constant = prev_container.constants.find do |const| + const.name == name_t.name + end + + if constant then + parse_method_dummy prev_container + return + end + end + + unless container then + # TODO seems broken, should starting at Object in @store + obj = name_t.name.split("::").inject(Object) do |state, item| + state.const_get(item) + end rescue nil + + type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule + + unless [Class, Module].include?(obj.class) then + warn("Couldn't find #{name_t.name}. Assuming it's a module") + end + + if type == RDoc::NormalClass then + sclass = obj.superclass ? obj.superclass.name : nil + container = prev_container.add_class type, name_t.name, sclass + else + container = prev_container.add_module type, name_t.name + end + + record_location container + end + + container + end + + ## + # Extracts a name or symbol from the token stream. + + def get_symbol_or_name + tk = get_tk + case tk + when TkSYMBOL then + text = tk.text.sub(/^:/, '') + + if TkASSIGN === peek_tk then + get_tk + text << '=' + end + + text + when TkId, TkOp then + tk.name + when TkAMPER, + TkDSTRING, + TkSTAR, + TkSTRING then + tk.text + else + raise RDoc::Error, "Name or symbol expected (got #{tk})" + end + end + + def stop_at_EXPR_END # :nodoc: + @scanner.lex_state == :EXPR_END || !@scanner.continue + end + + ## + # Marks containers between +container+ and +ancestor+ as ignored + + def suppress_parents container, ancestor # :nodoc: + while container and container != ancestor do + container.suppress unless container.documented? + container = container.parent + end + end + + ## + # Look for directives in a normal comment block: + # + # # :stopdoc: + # # Don't display comment from this point forward + # + # This routine modifies its +comment+ parameter. + + def look_for_directives_in context, comment + @preprocess.handle comment, context do |directive, param| + case directive + when 'method', 'singleton-method', + 'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then + false # handled elsewhere + when 'section' then + context.set_current_section param, comment.dup + comment.text = '' + break + end + end + + remove_private_comments comment + end + + ## + # Adds useful info about the parser to +message+ + + def make_message message + prefix = "#{@file_name}:" + + prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner + + "#{prefix} #{message}" + end + + ## + # Creates a comment with the correct format + + def new_comment comment + c = RDoc::Comment.new comment, @top_level + c.format = @markup + c + end + + ## + # Creates an RDoc::Attr for the name following +tk+, setting the comment to + # +comment+. + + def parse_attr(context, single, tk, comment) + offset = tk.seek + line_no = tk.line_no + + args = parse_symbol_arg 1 + if args.size > 0 then + name = args[0] + rw = "R" + skip_tkspace false + tk = get_tk + + if TkCOMMA === tk then + rw = "RW" if get_bool + else + unget_tk tk + end + + att = create_attr context, single, name, rw, comment + att.offset = offset + att.line = line_no + + read_documentation_modifiers att, RDoc::ATTR_MODIFIERS + else + warn "'attr' ignored - looks like a variable" + end + end + + ## + # Creates an RDoc::Attr for each attribute listed after +tk+, setting the + # comment for each to +comment+. + + def parse_attr_accessor(context, single, tk, comment) + offset = tk.seek + line_no = tk.line_no + + args = parse_symbol_arg + rw = "?" + + tmp = RDoc::CodeObject.new + read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS + # TODO In most other places we let the context keep track of document_self + # and add found items appropriately but here we do not. I'm not sure why. + return if @track_visibility and not tmp.document_self + + case tk.name + when "attr_reader" then rw = "R" + when "attr_writer" then rw = "W" + when "attr_accessor" then rw = "RW" + else + rw = '?' + end + + for name in args + att = create_attr context, single, name, rw, comment + att.offset = offset + att.line = line_no + end + end + + ## + # Parses an +alias+ in +context+ with +comment+ + + def parse_alias(context, single, tk, comment) + offset = tk.seek + line_no = tk.line_no + + skip_tkspace + + if TkLPAREN === peek_tk then + get_tk + skip_tkspace + end + + new_name = get_symbol_or_name + + @scanner.lex_state = :EXPR_FNAME + + skip_tkspace + if TkCOMMA === peek_tk then + get_tk + skip_tkspace + end + + begin + old_name = get_symbol_or_name + rescue RDoc::Error + return + end + + al = RDoc::Alias.new(get_tkread, old_name, new_name, comment, + single == SINGLE) + record_location al + al.offset = offset + al.line = line_no + + read_documentation_modifiers al, RDoc::ATTR_MODIFIERS + context.add_alias al + @stats.add_alias al + + al + end + + ## + # Extracts call parameters from the token stream. + + def parse_call_parameters(tk) + end_token = case tk + when TkLPAREN, TkfLPAREN + TkRPAREN + when TkRPAREN + return "" + else + TkNL + end + nest = 0 + + loop do + case tk + when TkSEMICOLON + break + when TkLPAREN, TkfLPAREN + nest += 1 + when end_token + if end_token == TkRPAREN + nest -= 1 + break if @scanner.lex_state == :EXPR_END and nest <= 0 + else + break unless @scanner.continue + end + when TkCOMMENT, TkASSIGN, TkOPASGN + unget_tk(tk) + break + when nil then + break + end + tk = get_tk + end + + get_tkread_clean "\n", " " + end + + ## + # Parses a class in +context+ with +comment+ + + def parse_class container, single, tk, comment + offset = tk.seek + line_no = tk.line_no + + declaration_context = container + container, name_t, given_name = get_class_or_module container + + cls = + case name_t + when TkCONSTANT + parse_class_regular container, declaration_context, single, + name_t, given_name, comment + when TkLSHFT + case name = get_class_specification + when 'self', container.name + parse_statements container, SINGLE + return # don't update offset or line + else + parse_class_singleton container, name, comment + end + else + warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}" + return + end + + cls.offset = offset + cls.line = line_no + + cls + end + + ## + # Parses and creates a regular class + + def parse_class_regular container, declaration_context, single, # :nodoc: + name_t, given_name, comment + superclass = '::Object' + + if given_name =~ /^::/ then + declaration_context = @top_level + given_name = $' + end + + if TkLT === peek_tk then + get_tk + skip_tkspace + superclass = get_class_specification + superclass = '(unknown)' if superclass.empty? + end + + cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass + cls = declaration_context.add_class cls_type, given_name, superclass + cls.ignore unless container.document_children + + read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS + record_location cls + + cls.add_comment comment, @top_level + + @top_level.add_to_classes_or_modules cls + @stats.add_class cls + + suppress_parents container, declaration_context unless cls.document_self + + parse_statements cls + + cls + end + + ## + # Parses a singleton class in +container+ with the given +name+ and + # +comment+. + + def parse_class_singleton container, name, comment # :nodoc: + other = @store.find_class_named name + + unless other then + if name =~ /^::/ then + name = $' + container = @top_level + end + + other = container.add_module RDoc::NormalModule, name + record_location other + + # class << $gvar + other.ignore if name.empty? + + other.add_comment comment, @top_level + end + + # notify :nodoc: all if not a constant-named class/module + # (and remove any comment) + unless name =~ /\A(::)?[A-Z]/ then + other.document_self = nil + other.document_children = false + other.clear_comment + end + + @top_level.add_to_classes_or_modules other + @stats.add_class other + + read_documentation_modifiers other, RDoc::CLASS_MODIFIERS + parse_statements(other, SINGLE) + + other + end + + ## + # Parses a constant in +context+ with +comment+. If +ignore_constants+ is + # true, no found constants will be added to RDoc. + + def parse_constant container, tk, comment, ignore_constants = false + offset = tk.seek + line_no = tk.line_no + + name = tk.name + skip_tkspace false + + return unless name =~ /^\w+$/ + + eq_tk = get_tk + + if TkCOLON2 === eq_tk then + unget_tk eq_tk + unget_tk tk + + container, name_t, = get_class_or_module container, ignore_constants + + name = name_t.name + + eq_tk = get_tk + end + + unless TkASSIGN === eq_tk then + unget_tk eq_tk + return false + end + + if TkGT === peek_tk then + unget_tk eq_tk + return + end + + value = '' + con = RDoc::Constant.new name, value, comment + + body = parse_constant_body container, con + + return unless body + + value.replace body + record_location con + con.offset = offset + con.line = line_no + read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS + + @stats.add_constant con + container.add_constant con + + true + end + + def parse_constant_body container, constant # :nodoc: + nest = 0 + rhs_name = '' + + get_tkread + + tk = get_tk + + loop do + case tk + when TkSEMICOLON then + break if nest <= 0 + when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK, + TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then + nest += 1 + when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then + nest -= 1 + when TkCOMMENT then + if nest <= 0 and stop_at_EXPR_END then + unget_tk tk + break + else + unget_tk tk + read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS + end + when TkCONSTANT then + rhs_name << tk.name + + if nest <= 0 and TkNL === peek_tk then + create_module_alias container, constant, rhs_name + break + end + when TkNL then + if nest <= 0 and stop_at_EXPR_END then + unget_tk tk + break + end + when TkCOLON2, TkCOLON3 then + rhs_name << '::' + when nil then + break + end + tk = get_tk + end + + get_tkread_clean(/^[ \t]+/, '') + end + + ## + # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for + # :method: or :attr: directives in +comment+. + + def parse_comment container, tk, comment + return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc' + column = tk.char_no + offset = tk.seek + line_no = tk.line_no + + text = comment.text + + singleton = !!text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3') + + co = + if text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then + parse_comment_ghost container, text, $1, column, line_no, comment + elsif text.sub!(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then + parse_comment_attr container, $1, $3, comment + end + + if co then + co.singleton = singleton + co.offset = offset + co.line = line_no + end + + true + end + + ## + # Parse a comment that is describing an attribute in +container+ with the + # given +name+ and +comment+. + + def parse_comment_attr container, type, name, comment # :nodoc: + return if name.empty? + + rw = case type + when 'attr_reader' then 'R' + when 'attr_writer' then 'W' + else 'RW' + end + + create_attr container, NORMAL, name, rw, comment + end + + def parse_comment_ghost container, text, name, column, line_no, # :nodoc: + comment + name = nil if name.empty? + + meth = RDoc::GhostMethod.new get_tkread, name + record_location meth + + meth.start_collecting_tokens + indent = TkSPACE.new 0, 1, 1 + indent.set_text " " * column + + position_comment = TkCOMMENT.new 0, line_no, 1 + position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}" + meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + + meth.params = + if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then + $1 + else + '' + end + + comment.normalize + comment.extract_call_seq meth + + return unless meth.name + + container.add_method meth + + meth.comment = comment + + @stats.add_method meth + + meth + end + + ## + # Creates an RDoc::Method on +container+ from +comment+ if there is a + # Signature section in the comment + + def parse_comment_tomdoc container, tk, comment + return unless signature = RDoc::TomDoc.signature(comment) + offset = tk.seek + line_no = tk.line_no + + name, = signature.split %r%[ \(]%, 2 + + meth = RDoc::GhostMethod.new get_tkread, name + record_location meth + meth.offset = offset + meth.line = line_no + + meth.start_collecting_tokens + indent = TkSPACE.new 0, 1, 1 + indent.set_text " " * offset + + position_comment = TkCOMMENT.new 0, line_no, 1 + position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}" + meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + + meth.call_seq = signature + + comment.normalize + + return unless meth.name + + container.add_method meth + + meth.comment = comment + + @stats.add_method meth + end + + ## + # Parses an +include+ or +extend+, indicated by the +klass+ and adds it to + # +container+ # with +comment+ + + def parse_extend_or_include klass, container, comment # :nodoc: + loop do + skip_tkspace_comment + + name = get_constant_with_optional_parens + + unless name.empty? then + obj = container.add klass, name, comment + record_location obj + end + + return unless TkCOMMA === peek_tk + + get_tk + end + end + + ## + # Parses identifiers that can create new methods or change visibility. + # + # Returns true if the comment was not consumed. + + def parse_identifier container, single, tk, comment # :nodoc: + case tk.name + when 'private', 'protected', 'public', 'private_class_method', + 'public_class_method', 'module_function' then + parse_visibility container, single, tk + return true + when 'attr' then + parse_attr container, single, tk, comment + when /^attr_(reader|writer|accessor)$/ then + parse_attr_accessor container, single, tk, comment + when 'alias_method' then + parse_alias container, single, tk, comment + when 'require', 'include' then + # ignore + else + if comment.text =~ /\A#\#$/ then + case comment.text + when /^# +:?attr(_reader|_writer|_accessor)?:/ then + parse_meta_attr container, single, tk, comment + else + method = parse_meta_method container, single, tk, comment + method.params = container.params if + container.params + method.block_params = container.block_params if + container.block_params + end + end + end + + false + end + + ## + # Parses a meta-programmed attribute and creates an RDoc::Attr. + # + # To create foo and bar attributes on class C with comment "My attributes": + # + # class C + # + # ## + # # :attr: + # # + # # My attributes + # + # my_attr :foo, :bar + # + # end + # + # To create a foo attribute on class C with comment "My attribute": + # + # class C + # + # ## + # # :attr: foo + # # + # # My attribute + # + # my_attr :foo, :bar + # + # end + + def parse_meta_attr(context, single, tk, comment) + args = parse_symbol_arg + rw = "?" + + # If nodoc is given, don't document any of them + + tmp = RDoc::CodeObject.new + read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS + + if comment.text.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then + rw = case $1 + when 'attr_reader' then 'R' + when 'attr_writer' then 'W' + else 'RW' + end + name = $3 unless $3.empty? + end + + if name then + att = create_attr context, single, name, rw, comment + else + args.each do |attr_name| + att = create_attr context, single, attr_name, rw, comment + end + end + + att + end + + ## + # Parses a meta-programmed method + + def parse_meta_method(container, single, tk, comment) + column = tk.char_no + offset = tk.seek + line_no = tk.line_no + + start_collecting_tokens + add_token tk + add_token_listener self + + skip_tkspace false + + singleton = !!comment.text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3') + + name = parse_meta_method_name comment, tk + + return unless name + + meth = RDoc::MetaMethod.new get_tkread, name + record_location meth + meth.offset = offset + meth.line = line_no + meth.singleton = singleton + + remove_token_listener self + + meth.start_collecting_tokens + indent = TkSPACE.new 0, 1, 1 + indent.set_text " " * column + + position_comment = TkCOMMENT.new 0, line_no, 1 + position_comment.value = "# File #{@top_level.relative_name}, line #{line_no}" + meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + meth.add_tokens @token_stream + + parse_meta_method_params container, single, meth, tk, comment + + meth.comment = comment + + @stats.add_method meth + + meth + end + + ## + # Parses the name of a metaprogrammed method. +comment+ is used to + # determine the name while +tk+ is used in an error message if the name + # cannot be determined. + + def parse_meta_method_name comment, tk # :nodoc: + if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then + return $1 unless $1.empty? + end + + name_t = get_tk + + case name_t + when TkSYMBOL then + name_t.text[1..-1] + when TkSTRING then + name_t.value[1..-2] + when TkASSIGN then # ignore + remove_token_listener self + + nil + else + warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'" + 'unknown' + end + end + + ## + # Parses the parameters and block for a meta-programmed method. + + def parse_meta_method_params container, single, meth, tk, comment # :nodoc: + token_listener meth do + meth.params = '' + + comment.normalize + comment.extract_call_seq meth + + container.add_method meth + + last_tk = tk + + while tk = get_tk do + case tk + when TkSEMICOLON then + break + when TkNL then + break unless last_tk and TkCOMMA === last_tk + when TkSPACE then + # expression continues + when TkDO then + parse_statements container, single, meth + break + else + last_tk = tk + end + end + end + end + + ## + # Parses a normal method defined by +def+ + + def parse_method(container, single, tk, comment) + singleton = nil + added_container = false + name = nil + column = tk.char_no + offset = tk.seek + line_no = tk.line_no + + start_collecting_tokens + add_token tk + + token_listener self do + prev_container = container + name, container, singleton = parse_method_name container + added_container = container != prev_container + end + + return unless name + + meth = RDoc::AnyMethod.new get_tkread, name + meth.singleton = single == SINGLE ? true : singleton + + record_location meth + meth.offset = offset + meth.line = line_no + + meth.start_collecting_tokens + indent = TkSPACE.new 0, 1, 1 + indent.set_text " " * column + + token = TkCOMMENT.new 0, line_no, 1 + token.set_text "# File #{@top_level.relative_name}, line #{line_no}" + meth.add_tokens [token, NEWLINE_TOKEN, indent] + meth.add_tokens @token_stream + + parse_method_params_and_body container, single, meth, added_container + + comment.normalize + comment.extract_call_seq meth + + meth.comment = comment + + @stats.add_method meth + end + + ## + # Parses the parameters and body of +meth+ + + def parse_method_params_and_body container, single, meth, added_container + token_listener meth do + @scanner.continue = false + parse_method_parameters meth + + if meth.document_self or not @track_visibility then + container.add_method meth + elsif added_container then + container.document_self = false + end + + # Having now read the method parameters and documentation modifiers, we + # now know whether we have to rename #initialize to ::new + + if meth.name == "initialize" && !meth.singleton then + if meth.dont_rename_initialize then + meth.visibility = :protected + else + meth.singleton = true + meth.name = "new" + meth.visibility = :public + end + end + + parse_statements container, single, meth + end + end + + ## + # Parses a method that needs to be ignored. + + def parse_method_dummy container + dummy = RDoc::Context.new + dummy.parent = container + dummy.store = container.store + skip_method dummy + end + + ## + # Parses the name of a method in +container+. + # + # Returns the method name, the container it is in (for def Foo.name) and if + # it is a singleton or regular method. + + def parse_method_name container # :nodoc: + @scanner.lex_state = :EXPR_FNAME + + skip_tkspace + name_t = get_tk + back_tk = skip_tkspace + singleton = false + + case dot = get_tk + when TkDOT, TkCOLON2 then + singleton = true + + name, container = parse_method_name_singleton container, name_t + else + unget_tk dot + back_tk.reverse_each do |token| + unget_tk token + end + + name = parse_method_name_regular container, name_t + end + + return name, container, singleton + end + + ## + # For the given +container+ and initial name token +name_t+ the method name + # is parsed from the token stream for a regular method. + + def parse_method_name_regular container, name_t # :nodoc: + case name_t + when TkSTAR, TkAMPER then + name_t.text + else + unless name_t.respond_to? :name then + warn "expected method name token, . or ::, got #{name_t.inspect}" + skip_method container + return + end + name_t.name + end + end + + ## + # For the given +container+ and initial name token +name_t+ the method name + # and the new +container+ (if necessary) are parsed from the token stream + # for a singleton method. + + def parse_method_name_singleton container, name_t # :nodoc: + @scanner.lex_state = :EXPR_FNAME + skip_tkspace + name_t2 = get_tk + + name = + case name_t + when TkSELF, TkMOD then + case name_t2 + # NOTE: work around '[' being consumed early and not being re-tokenized + # as a TkAREF + when TkfLBRACK then + get_tk + '[]' + else + name_t2.name + end + when TkCONSTANT then + name = name_t2.name + + container = get_method_container container, name_t + + return unless container + + name + when TkIDENTIFIER, TkIVAR, TkGVAR then + parse_method_dummy container + + nil + when TkTRUE, TkFALSE, TkNIL then + klass_name = "#{name_t.name.capitalize}Class" + container = @store.find_class_named klass_name + container ||= @top_level.add_class RDoc::NormalClass, klass_name + + name_t2.name + else + warn "unexpected method name token #{name_t.inspect}" + # break + skip_method container + + nil + end + + return name, container + end + + ## + # Extracts +yield+ parameters from +method+ + + def parse_method_or_yield_parameters(method = nil, + modifiers = RDoc::METHOD_MODIFIERS) + skip_tkspace false + tk = get_tk + end_token = get_end_token tk + return '' unless end_token + + nest = 0 + + loop do + case tk + when TkSEMICOLON then + break if nest == 0 + when TkLBRACE, TkfLBRACE then + nest += 1 + when TkRBRACE then + nest -= 1 + if nest <= 0 + # we might have a.each { |i| yield i } + unget_tk(tk) if nest < 0 + break + end + when TkLPAREN, TkfLPAREN then + nest += 1 + when end_token then + if end_token == TkRPAREN + nest -= 1 + break if nest <= 0 + else + break unless @scanner.continue + end + when TkRPAREN then + nest -= 1 + when method && method.block_params.nil? && TkCOMMENT then + unget_tk tk + read_documentation_modifiers method, modifiers + @read.pop + when TkCOMMENT then + @read.pop + when nil then + break + end + tk = get_tk + end + + get_tkread_clean(/\s+/, ' ') + end + + ## + # Capture the method's parameters. Along the way, look for a comment + # containing: + # + # # yields: .... + # + # and add this as the block_params for the method + + def parse_method_parameters method + res = parse_method_or_yield_parameters method + + res = "(#{res})" unless res =~ /\A\(/ + method.params = res unless method.params + + return if method.block_params + + skip_tkspace false + read_documentation_modifiers method, RDoc::METHOD_MODIFIERS + end + + ## + # Parses an RDoc::NormalModule in +container+ with +comment+ + + def parse_module container, single, tk, comment + container, name_t, = get_class_or_module container + + name = name_t.name + + mod = container.add_module RDoc::NormalModule, name + mod.ignore unless container.document_children + record_location mod + + read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS + mod.add_comment comment, @top_level + parse_statements mod + + @stats.add_module mod + end + + ## + # Parses an RDoc::Require in +context+ containing +comment+ + + def parse_require(context, comment) + skip_tkspace_comment + tk = get_tk + + if TkLPAREN === tk then + skip_tkspace_comment + tk = get_tk + end + + name = tk.text if TkSTRING === tk + + if name then + @top_level.add_require RDoc::Require.new(name, comment) + else + unget_tk tk + end + end + + ## + # Parses a rescue + + def parse_rescue + skip_tkspace false + + while tk = get_tk + case tk + when TkNL, TkSEMICOLON then + break + when TkCOMMA then + skip_tkspace false + + get_tk if TkNL === peek_tk + end + + skip_tkspace false + end + end + + ## + # The core of the Ruby parser. + + def parse_statements(container, single = NORMAL, current_method = nil, + comment = new_comment('')) + raise 'no' unless RDoc::Comment === comment + comment.force_encoding @encoding if @encoding + + nest = 1 + save_visibility = container.visibility + + non_comment_seen = true + + while tk = get_tk do + keep_comment = false + try_parse_comment = false + + non_comment_seen = true unless TkCOMMENT === tk + + case tk + when TkNL then + skip_tkspace + tk = get_tk + + if TkCOMMENT === tk then + if non_comment_seen then + # Look for RDoc in a comment about to be thrown away + non_comment_seen = parse_comment container, tk, comment unless + comment.empty? + + comment = '' + comment.force_encoding @encoding if @encoding + end + + while TkCOMMENT === tk do + comment << tk.text << "\n" + + tk = get_tk + + if TkNL === tk then + skip_tkspace false # leading spaces + tk = get_tk + end + end + + comment = new_comment comment + + unless comment.empty? then + look_for_directives_in container, comment + + if container.done_documenting then + throw :eof if RDoc::TopLevel === container + container.ongoing_visibility = save_visibility + end + end + + keep_comment = true + else + non_comment_seen = true + end + + unget_tk tk + keep_comment = true + + when TkCLASS then + parse_class container, single, tk, comment + + when TkMODULE then + parse_module container, single, tk, comment + + when TkDEF then + parse_method container, single, tk, comment + + when TkCONSTANT then + unless parse_constant container, tk, comment, current_method then + try_parse_comment = true + end + + when TkALIAS then + parse_alias container, single, tk, comment unless current_method + + when TkYIELD then + if current_method.nil? then + warn "Warning: yield outside of method" if container.document_self + else + parse_yield container, single, tk, current_method + end + + # Until and While can have a 'do', which shouldn't increase the nesting. + # We can't solve the general case, but we can handle most occurrences by + # ignoring a do at the end of a line. + + when TkUNTIL, TkWHILE then + nest += 1 + skip_optional_do_after_expression + + # 'for' is trickier + when TkFOR then + nest += 1 + skip_for_variable + skip_optional_do_after_expression + + when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then + nest += 1 + + when TkSUPER then + current_method.calls_super = true if current_method + + when TkRESCUE then + parse_rescue + + when TkIDENTIFIER then + if nest == 1 and current_method.nil? then + keep_comment = parse_identifier container, single, tk, comment + end + + case tk.name + when "require" then + parse_require container, comment + when "include" then + parse_extend_or_include RDoc::Include, container, comment + when "extend" then + parse_extend_or_include RDoc::Extend, container, comment + end + + when TkEND then + nest -= 1 + if nest == 0 then + read_documentation_modifiers container, RDoc::CLASS_MODIFIERS + container.ongoing_visibility = save_visibility + + parse_comment container, tk, comment unless comment.empty? + + return + end + else + try_parse_comment = nest == 1 + end + + if try_parse_comment then + non_comment_seen = parse_comment container, tk, comment unless + comment.empty? + + keep_comment = false + end + + unless keep_comment then + comment = new_comment '' + comment.force_encoding @encoding if @encoding + container.params = nil + container.block_params = nil + end + + consume_trailing_spaces + end + + container.params = nil + container.block_params = nil + end + + ## + # Parse up to +no+ symbol arguments + + def parse_symbol_arg(no = nil) + skip_tkspace_comment + + case tk = get_tk + when TkLPAREN + parse_symbol_arg_paren no + else + parse_symbol_arg_space no, tk + end + end + + ## + # Parses up to +no+ symbol arguments surrounded by () and places them in + # +args+. + + def parse_symbol_arg_paren no # :nodoc: + args = [] + + loop do + skip_tkspace_comment + if tk1 = parse_symbol_in_arg + args.push tk1 + break if no and args.size >= no + end + + skip_tkspace_comment + case tk2 = get_tk + when TkRPAREN + break + when TkCOMMA + else + warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC + break + end + end + + args + end + + ## + # Parses up to +no+ symbol arguments separated by spaces and places them in + # +args+. + + def parse_symbol_arg_space no, tk # :nodoc: + args = [] + + unget_tk tk + if tk = parse_symbol_in_arg + args.push tk + return args if no and args.size >= no + end + + loop do + skip_tkspace false + + tk1 = get_tk + unless TkCOMMA === tk1 then + unget_tk tk1 + break + end + + skip_tkspace_comment + if tk = parse_symbol_in_arg + args.push tk + break if no and args.size >= no + end + end + + args + end + + ## + # Returns symbol text from the next token + + def parse_symbol_in_arg + case tk = get_tk + when TkSYMBOL + tk.text.sub(/^:/, '') + when TkSTRING + eval @read[-1] + when TkDSTRING, TkIDENTIFIER then + nil # ignore + else + warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC + nil + end + end + + ## + # Parses statements in the top-level +container+ + + def parse_top_level_statements container + comment = collect_first_comment + + look_for_directives_in container, comment + + throw :eof if container.done_documenting + + @markup = comment.format + + # HACK move if to RDoc::Context#comment= + container.comment = comment if container.document_self unless comment.empty? + + parse_statements container, NORMAL, nil, comment + end + + ## + # Determines the visibility in +container+ from +tk+ + + def parse_visibility(container, single, tk) + vis_type, vis, singleton = get_visibility_information tk, single + + skip_tkspace_comment false + + case peek_tk + # Ryan Davis suggested the extension to ignore modifiers, because he + # often writes + # + # protected unless $TESTING + # + when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then + container.ongoing_visibility = vis + else + update_visibility container, vis_type, vis, singleton + end + end + + ## + # Determines the block parameter for +context+ + + def parse_yield(context, single, tk, method) + return if method.block_params + + get_tkread + @scanner.continue = false + method.block_params = parse_method_or_yield_parameters + end + + ## + # Directives are modifier comments that can appear after class, module, or + # method names. For example: + # + # def fred # :yields: a, b + # + # or: + # + # class MyClass # :nodoc: + # + # We return the directive name and any parameters as a two element array if + # the name is in +allowed+. A directive can be found anywhere up to the end + # of the current line. + + def read_directive allowed + tokens = [] + + while tk = get_tk do + tokens << tk + + case tk + when TkNL, TkDEF then + return + when TkCOMMENT then + return unless tk.text =~ /\s*:?([\w-]+):\s*(.*)/ + + directive = $1.downcase + + return [directive, $2] if allowed.include? directive + + return + end + end + ensure + unless tokens.length == 1 and TkCOMMENT === tokens.first then + tokens.reverse_each do |token| + unget_tk token + end + end + end + + ## + # Handles directives following the definition for +context+ (any + # RDoc::CodeObject) if the directives are +allowed+ at this point. + # + # See also RDoc::Markup::PreProcess#handle_directive + + def read_documentation_modifiers context, allowed + directive, value = read_directive allowed + + return unless directive + + @preprocess.handle_directive '', directive, value, context do |dir, param| + if %w[notnew not_new not-new].include? dir then + context.dont_rename_initialize = true + + true + end + end + end + + ## + # Records the location of this +container+ in the file for this parser and + # adds it to the list of classes and modules in the file. + + def record_location container # :nodoc: + case container + when RDoc::ClassModule then + @top_level.add_to_classes_or_modules container + end + + container.record_location @top_level + end + + ## + # Removes private comments from +comment+ + #-- + # TODO remove + + def remove_private_comments comment + comment.remove_private + end + + ## + # Scans this Ruby file for Ruby constructs + + def scan + reset + + catch :eof do + begin + parse_top_level_statements @top_level + + rescue StandardError => e + bytes = '' + + 20.times do @scanner.ungetc end + count = 0 + 60.times do |i| + count = i + byte = @scanner.getc + break unless byte + bytes << byte + end + count -= 20 + count.times do @scanner.ungetc end + + $stderr.puts <<-EOF + +#{self.class} failure around line #{@scanner.line_no} of +#{@file_name} + + EOF + + unless bytes.empty? then + $stderr.puts + $stderr.puts bytes.inspect + end + + raise e + end + end + + @top_level + end + + ## + # while, until, and for have an optional do + + def skip_optional_do_after_expression + skip_tkspace false + tk = get_tk + end_token = get_end_token tk + + b_nest = 0 + nest = 0 + @scanner.continue = false + + loop do + case tk + when TkSEMICOLON then + break if b_nest.zero? + when TkLPAREN, TkfLPAREN then + nest += 1 + when TkBEGIN then + b_nest += 1 + when TkEND then + b_nest -= 1 + when TkDO + break if nest.zero? + when end_token then + if end_token == TkRPAREN + nest -= 1 + break if @scanner.lex_state == :EXPR_END and nest.zero? + else + break unless @scanner.continue + end + when nil then + break + end + tk = get_tk + end + + skip_tkspace false + + get_tk if TkDO === peek_tk + end + + ## + # skip the var [in] part of a 'for' statement + + def skip_for_variable + skip_tkspace false + get_tk + skip_tkspace false + tk = get_tk + unget_tk(tk) unless TkIN === tk + end + + ## + # Skips the next method in +container+ + + def skip_method container + meth = RDoc::AnyMethod.new "", "anon" + parse_method_parameters meth + parse_statements container, false, meth + end + + ## + # Skip spaces until a comment is found + + def skip_tkspace_comment(skip_nl = true) + loop do + skip_tkspace skip_nl + return unless TkCOMMENT === peek_tk + get_tk + end + end + + ## + # Updates visibility in +container+ from +vis_type+ and +vis+. + + def update_visibility container, vis_type, vis, singleton # :nodoc: + new_methods = [] + + case vis_type + when 'module_function' then + args = parse_symbol_arg + container.set_visibility_for args, :private, false + + container.methods_matching args do |m| + s_m = m.dup + record_location s_m + s_m.singleton = true + new_methods << s_m + end + when 'public_class_method', 'private_class_method' then + args = parse_symbol_arg + + container.methods_matching args, true do |m| + if m.parent != container then + m = m.dup + record_location m + new_methods << m + end + + m.visibility = vis + end + else + args = parse_symbol_arg + container.set_visibility_for args, vis, singleton + end + + new_methods.each do |method| + case method + when RDoc::AnyMethod then + container.add_method method + when RDoc::Attr then + container.add_attribute method + end + method.visibility = vis + end + end + + ## + # Prints +message+ to +$stderr+ unless we're being quiet + + def warn message + @options.warn make_message message + end + +end + diff --git a/jni/ruby/lib/rdoc/parser/ruby_tools.rb b/jni/ruby/lib/rdoc/parser/ruby_tools.rb new file mode 100644 index 0000000..654431e --- /dev/null +++ b/jni/ruby/lib/rdoc/parser/ruby_tools.rb @@ -0,0 +1,167 @@ +## +# Collection of methods for writing parsers against RDoc::RubyLex and +# RDoc::RubyToken + +module RDoc::Parser::RubyTools + + include RDoc::RubyToken + + ## + # Adds a token listener +obj+, but you should probably use token_listener + + def add_token_listener(obj) + @token_listeners ||= [] + @token_listeners << obj + end + + ## + # Fetches the next token from the scanner + + def get_tk + tk = nil + + if @tokens.empty? then + tk = @scanner.token + @read.push @scanner.get_readed + puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG + else + @read.push @unget_read.shift + tk = @tokens.shift + puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG + end + + tk = nil if TkEND_OF_SCRIPT === tk + + if TkSYMBEG === tk then + set_token_position tk.line_no, tk.char_no + + case tk1 = get_tk + when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then + if tk1.respond_to?(:name) then + tk = Token(TkSYMBOL).set_text(":" + tk1.name) + else + tk = Token(TkSYMBOL).set_text(":" + tk1.text) + end + + # remove the identifier we just read to replace it with a symbol + @token_listeners.each do |obj| + obj.pop_token + end if @token_listeners + else + tk = tk1 + end + end + + # inform any listeners of our shiny new token + @token_listeners.each do |obj| + obj.add_token(tk) + end if @token_listeners + + tk + end + + ## + # Reads and returns all tokens up to one of +tokens+. Leaves the matched + # token in the token list. + + def get_tk_until(*tokens) + read = [] + + loop do + tk = get_tk + + case tk + when *tokens then + unget_tk tk + break + end + + read << tk + end + + read + end + + ## + # Retrieves a String representation of the read tokens + + def get_tkread + read = @read.join("") + @read = [] + read + end + + ## + # Peek equivalent for get_tkread + + def peek_read + @read.join('') + end + + ## + # Peek at the next token, but don't remove it from the stream + + def peek_tk + unget_tk(tk = get_tk) + tk + end + + ## + # Removes the token listener +obj+ + + def remove_token_listener(obj) + @token_listeners.delete(obj) + end + + ## + # Resets the tools + + def reset + @read = [] + @tokens = [] + @unget_read = [] + @nest = 0 + end + + ## + # Skips whitespace tokens including newlines if +skip_nl+ is true + + def skip_tkspace(skip_nl = true) # HACK dup + tokens = [] + + while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do + tokens.push tk + end + + unget_tk tk + tokens + end + + ## + # Has +obj+ listen to tokens + + def token_listener(obj) + add_token_listener obj + yield + ensure + remove_token_listener obj + end + + ## + # Returns +tk+ to the scanner + + def unget_tk(tk) + @tokens.unshift tk + @unget_read.unshift @read.pop + + # Remove this token from any listeners + @token_listeners.each do |obj| + obj.pop_token + end if @token_listeners + + nil + end + +end + + diff --git a/jni/ruby/lib/rdoc/parser/simple.rb b/jni/ruby/lib/rdoc/parser/simple.rb new file mode 100644 index 0000000..65cfc1b --- /dev/null +++ b/jni/ruby/lib/rdoc/parser/simple.rb @@ -0,0 +1,61 @@ +## +# Parse a non-source file. We basically take the whole thing as one big +# comment. + +class RDoc::Parser::Simple < RDoc::Parser + + include RDoc::Parser::Text + + parse_files_matching(//) + + attr_reader :content # :nodoc: + + ## + # Prepare to parse a plain file + + def initialize(top_level, file_name, content, options, stats) + super + + preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include + + preprocess.handle @content, @top_level + end + + ## + # Extract the file contents and attach them to the TopLevel as a comment + + def scan + comment = remove_coding_comment @content + comment = remove_private_comment comment + + comment = RDoc::Comment.new comment, @top_level + + @top_level.comment = comment + @top_level + end + + ## + # Removes the encoding magic comment from +text+ + + def remove_coding_comment text + text.sub(/\A# .*coding[=:].*$/, '') + end + + ## + # Removes private comments. + # + # Unlike RDoc::Comment#remove_private this implementation only looks for two + # dashes at the beginning of the line. Three or more dashes are considered + # to be a rule and ignored. + + def remove_private_comment comment + # Workaround for gsub encoding for Ruby 1.9.2 and earlier + empty = '' + empty.force_encoding comment.encoding if Object.const_defined? :Encoding + + comment = comment.gsub(%r%^--\n.*?^\+\+\n?%m, empty) + comment.sub(%r%^--\n.*%m, empty) + end + +end + diff --git a/jni/ruby/lib/rdoc/parser/text.rb b/jni/ruby/lib/rdoc/parser/text.rb new file mode 100644 index 0000000..f973313 --- /dev/null +++ b/jni/ruby/lib/rdoc/parser/text.rb @@ -0,0 +1,11 @@ +## +# Indicates this parser is text and doesn't contain code constructs. +# +# Include this module in a RDoc::Parser subclass to make it show up as a file, +# not as part of a class or module. +#-- +# This is not named File to avoid overriding ::File + +module RDoc::Parser::Text +end + |