Fresh start

5 files changed, 524 insertions, 0 deletions

diff --git a/jni/ruby/ext/ripper/lib/ripper.rb b/jni/ruby/ext/ripper/lib/ripper.rb
new file mode 100644
index 0000000..542bd40
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper.rb
@@ -0,0 +1,73 @@
+require 'ripper/core'
+require 'ripper/lexer'
+require 'ripper/filter'
+require 'ripper/sexp'
+
+# Ripper is a Ruby script parser.
+#
+# You can get information from the parser with event-based style.
+# Information such as abstract syntax trees or simple lexical analysis of the
+# Ruby program.
+#
+# == Usage
+#
+# Ripper provides an easy interface for parsing your program into a symbolic
+# expression tree (or S-expression).
+#
+# Understanding the output of the parser may come as a challenge, it's
+# recommended you use PP to format the output for legibility.
+#
+#   require 'ripper'
+#   require 'pp'
+#
+#   pp Ripper.sexp('def hello(world) "Hello, #{world}!"; end')
+#     #=> [:program,
+#          [[:def,
+#            [:@ident, "hello", [1, 4]],
+#            [:paren,
+#             [:params, [[:@ident, "world", [1, 10]]], nil, nil, nil, nil, nil, nil]],
+#            [:bodystmt,
+#             [[:string_literal,
+#               [:string_content,
+#                [:@tstring_content, "Hello, ", [1, 18]],
+#                [:string_embexpr, [[:var_ref, [:@ident, "world", [1, 27]]]]],
+#                [:@tstring_content, "!", [1, 33]]]]],
+#             nil,
+#             nil,
+#             nil]]]]
+#
+# You can see in the example above, the expression starts with +:program+.
+#
+# From here, a method definition at +:def+, followed by the method's identifier
+# <code>:@ident</code>. After the method's identifier comes the parentheses
+# +:paren+ and the method parameters under +:params+.
+#
+# Next is the method body, starting at +:bodystmt+ (+stmt+ meaning statement),
+# which contains the full definition of the method.
+#
+# In our case, we're simply returning a String, so next we have the
+# +:string_literal+ expression.
+#
+# Within our +:string_literal+ you'll notice two <code>@tstring_content</code>,
+# this is the literal part for <code>Hello, </code> and <code>!</code>. Between
+# the two <code>@tstring_content</code> statements is a +:string_embexpr+,
+# where _embexpr_ is an embedded expression. Our expression consists of a local
+# variable, or +var_ref+, with the identifier (<code>@ident</code>) of +world+.
+#
+# == Resources
+#
+# * {Ruby Inside}[http://www.rubyinside.com/using-ripper-to-see-how-ruby-is-parsing-your-code-5270.html]
+#
+# == Requirements
+#
+# * ruby 1.9 (support CVS HEAD only)
+# * bison 1.28 or later (Other yaccs do not work)
+#
+# == License
+#
+#   Ruby License.
+#
+#                                                 Minero Aoki
+#                                         aamine@loveruby.net
+#                                       http://i.loveruby.net
+class Ripper; end
diff --git a/jni/ruby/ext/ripper/lib/ripper/core.rb b/jni/ruby/ext/ripper/lib/ripper/core.rb
new file mode 100644
index 0000000..008ec53
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper/core.rb
@@ -0,0 +1,70 @@
+#
+# $Id: core.rb 36954 2012-09-12 23:04:41Z zzak $
+#
+# Copyright (c) 2003-2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper.so'
+
+class Ripper
+
+  # Parses the given Ruby program read from +src+.
+  # +src+ must be a String or an IO or a object with a #gets method.
+  def Ripper.parse(src, filename = '(ripper)', lineno = 1)
+    new(src, filename, lineno).parse
+  end
+
+  # This array contains name of parser events.
+  PARSER_EVENTS = PARSER_EVENT_TABLE.keys
+
+  # This array contains name of scanner events.
+  SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
+
+  # This array contains name of all ripper events.
+  EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+  private
+
+  #
+  # Parser Events
+  #
+
+  PARSER_EVENT_TABLE.each do |id, arity|
+    module_eval(<<-End, __FILE__, __LINE__ + 1)
+      def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') })
+        #{arity == 0 ? 'nil' : 'a'}
+      end
+    End
+  end
+
+  # This method is called when weak warning is produced by the parser.
+  # +fmt+ and +args+ is printf style.
+  def warn(fmt, *args)
+  end
+
+  # This method is called when strong warning is produced by the parser.
+  # +fmt+ and +args+ is printf style.
+  def warning(fmt, *args)
+  end
+
+  # This method is called when the parser found syntax error.
+  def compile_error(msg)
+  end
+
+  #
+  # Scanner Events
+  #
+
+  SCANNER_EVENTS.each do |id|
+    module_eval(<<-End, __FILE__, __LINE__ + 1)
+      def on_#{id}(token)
+        token
+      end
+    End
+  end
+
+end
diff --git a/jni/ruby/ext/ripper/lib/ripper/filter.rb b/jni/ruby/ext/ripper/lib/ripper/filter.rb
new file mode 100644
index 0000000..b30b085
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper/filter.rb
@@ -0,0 +1,77 @@
+#
+# $Id: filter.rb 36954 2012-09-12 23:04:41Z zzak $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/lexer'
+
+class Ripper
+
+  # This class handles only scanner events,
+  # which are dispatched in the 'right' order (same with input).
+  class Filter
+
+    # Creates a new Ripper::Filter instance, passes parameters +src+,
+    # +filename+, and +lineno+ to Ripper::Lexer.new
+    #
+    # The lexer is for internal use only.
+    def initialize(src, filename = '-', lineno = 1)
+      @__lexer = Lexer.new(src, filename, lineno)
+      @__line = nil
+      @__col = nil
+    end
+
+    # The file name of the input.
+    def filename
+      @__lexer.filename
+    end
+
+    # The line number of the current token.
+    # This value starts from 1.
+    # This method is valid only in event handlers.
+    def lineno
+      @__line
+    end
+
+    # The column number of the current token.
+    # This value starts from 0.
+    # This method is valid only in event handlers.
+    def column
+      @__col
+    end
+
+    # Starts the parser.
+    # +init+ is a data accumulator and is passed to the next event handler (as
+    # of Enumerable#inject).
+    def parse(init = nil)
+      data = init
+      @__lexer.lex.each do |pos, event, tok|
+        @__line, @__col = *pos
+        data = if respond_to?(event, true)
+               then __send__(event, tok, data)
+               else on_default(event, tok, data)
+               end
+      end
+      data
+    end
+
+    private
+
+    # This method is called when some event handler is undefined.
+    # +event+ is :on_XXX, +token+ is the scanned token, and +data+ is a data
+    # accumulator.
+    #
+    # The return value of this method is passed to the next event handler (as
+    # of Enumerable#inject).
+    def on_default(event, token, data)
+      data
+    end
+
+  end
+
+end
diff --git a/jni/ruby/ext/ripper/lib/ripper/lexer.rb b/jni/ruby/ext/ripper/lib/ripper/lexer.rb
new file mode 100644
index 0000000..38b2087
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper/lexer.rb
@@ -0,0 +1,186 @@
+#
+# $Id: lexer.rb 44878 2014-02-07 12:57:44Z zzak $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/core'
+
+class Ripper
+
+  # Tokenizes the Ruby program and returns an array of strings.
+  #
+  #   p Ripper.tokenize("def m(a) nil end")
+  #      # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"]
+  #
+  def Ripper.tokenize(src, filename = '-', lineno = 1)
+    Lexer.new(src, filename, lineno).tokenize
+  end
+
+  # Tokenizes the Ruby program and returns an array of an array,
+  # which is formatted like <code>[[lineno, column], type, token]</code>.
+  #
+  #   require 'ripper'
+  #   require 'pp'
+  #
+  #   pp Ripper.lex("def m(a) nil end")
+  #     #=> [[[1,  0], :on_kw,     "def"],
+  #          [[1,  3], :on_sp,     " "  ],
+  #          [[1,  4], :on_ident,  "m"  ],
+  #          [[1,  5], :on_lparen, "("  ],
+  #          [[1,  6], :on_ident,  "a"  ],
+  #          [[1,  7], :on_rparen, ")"  ],
+  #          [[1,  8], :on_sp,     " "  ],
+  #          [[1,  9], :on_kw,     "nil"],
+  #          [[1, 12], :on_sp,     " "  ],
+  #          [[1, 13], :on_kw,     "end"]]
+  #
+  def Ripper.lex(src, filename = '-', lineno = 1)
+    Lexer.new(src, filename, lineno).lex
+  end
+
+  class Lexer < ::Ripper   #:nodoc: internal use only
+    def tokenize
+      lex().map {|pos, event, tok| tok }
+    end
+
+    def lex
+      parse().sort_by {|pos, event, tok| pos }
+    end
+
+    def parse
+      @buf = []
+      super
+      @buf
+    end
+
+    private
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
+        def on_#{event}(tok)
+          @buf.push [[lineno(), column()], :on_#{event}, tok]
+        end
+      End
+    end
+  end
+
+  # [EXPERIMENTAL]
+  # Parses +src+ and return a string which was matched to +pattern+.
+  # +pattern+ should be described as Regexp.
+  #
+  #   require 'ripper'
+  #
+  #   p Ripper.slice('def m(a) nil end', 'ident')                   #=> "m"
+  #   p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+')  #=> "m(a)"
+  #   p Ripper.slice("<<EOS\nstring\nEOS",
+  #                  'heredoc_beg nl $(tstring_content*) heredoc_end', 1)
+  #       #=> "string\n"
+  #
+  def Ripper.slice(src, pattern, n = 0)
+    if m = token_match(src, pattern)
+    then m.string(n)
+    else nil
+    end
+  end
+
+  def Ripper.token_match(src, pattern)   #:nodoc:
+    TokenPattern.compile(pattern).match(src)
+  end
+
+  class TokenPattern   #:nodoc:
+
+    class Error < ::StandardError # :nodoc:
+    end
+    class CompileError < Error # :nodoc:
+    end
+    class MatchError < Error # :nodoc:
+    end
+
+    class << self
+      alias compile new
+    end
+
+    def initialize(pattern)
+      @source = pattern
+      @re = compile(pattern)
+    end
+
+    def match(str)
+      match_list(::Ripper.lex(str))
+    end
+
+    def match_list(tokens)
+      if m = @re.match(map_tokens(tokens))
+      then MatchData.new(tokens, m)
+      else nil
+      end
+    end
+
+    private
+
+    def compile(pattern)
+      if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern)
+        raise CompileError, "invalid char in pattern: #{m[0].inspect}"
+      end
+      buf = ''
+      pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok|
+        case tok
+        when /\w/
+          buf.concat map_token(tok)
+        when '$('
+          buf.concat '('
+        when '('
+          buf.concat '(?:'
+        when /[?*\[\])\.]/
+          buf.concat tok
+        else
+          raise 'must not happen'
+        end
+      end
+      Regexp.compile(buf)
+    rescue RegexpError => err
+      raise CompileError, err.message
+    end
+
+    def map_tokens(tokens)
+      tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join
+    end
+
+    MAP = {}
+    seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
+    SCANNER_EVENT_TABLE.each do |ev, |
+      raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty?
+      MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift
+    end
+
+    def map_token(tok)
+      MAP[tok]  or raise CompileError, "unknown token: #{tok}"
+    end
+
+    class MatchData # :nodoc:
+      def initialize(tokens, match)
+        @tokens = tokens
+        @match = match
+      end
+
+      def string(n = 0)
+        return nil unless @match
+        match(n).join
+      end
+
+      private
+
+      def match(n = 0)
+        return [] unless @match
+        @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str }
+      end
+    end
+
+  end
+
+end
diff --git a/jni/ruby/ext/ripper/lib/ripper/sexp.rb b/jni/ruby/ext/ripper/lib/ripper/sexp.rb
new file mode 100644
index 0000000..514a585
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper/sexp.rb
@@ -0,0 +1,118 @@
+#
+# $Id: sexp.rb 48144 2014-10-26 03:24:18Z nobu $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/core'
+
+class Ripper
+
+  # [EXPERIMENTAL]
+  # Parses +src+ and create S-exp tree.
+  # Returns more readable tree rather than Ripper.sexp_raw.
+  # This method is mainly for developer use.
+  #
+  #   require 'ripper'
+  #   require 'pp'
+  #
+  #   pp Ripper.sexp("def m(a) nil end")
+  #     #=> [:program,
+  #          [[:def,
+  #           [:@ident, "m", [1, 4]],
+  #           [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil, nil]],
+  #           [:bodystmt, [[:var_ref, [:@kw, "nil", [1, 9]]]], nil, nil, nil]]]]
+  #
+  def Ripper.sexp(src, filename = '-', lineno = 1)
+    builder = SexpBuilderPP.new(src, filename, lineno)
+    sexp = builder.parse
+    sexp unless builder.error?
+  end
+
+  # [EXPERIMENTAL]
+  # Parses +src+ and create S-exp tree.
+  # This method is mainly for developer use.
+  #
+  #   require 'ripper'
+  #   require 'pp'
+  #
+  #   pp Ripper.sexp_raw("def m(a) nil end")
+  #     #=> [:program,
+  #          [:stmts_add,
+  #           [:stmts_new],
+  #           [:def,
+  #            [:@ident, "m", [1, 4]],
+  #            [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
+  #            [:bodystmt,
+  #             [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
+  #             nil,
+  #             nil,
+  #             nil]]]]
+  #
+  def Ripper.sexp_raw(src, filename = '-', lineno = 1)
+    builder = SexpBuilder.new(src, filename, lineno)
+    sexp = builder.parse
+    sexp unless builder.error?
+  end
+
+  class SexpBuilderPP < ::Ripper   #:nodoc:
+    private
+
+    PARSER_EVENT_TABLE.each do |event, arity|
+      if /_new\z/ =~ event.to_s and arity == 0
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}
+            []
+          end
+        End
+      elsif /_add\z/ =~ event.to_s
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(list, item)
+            list.push item
+            list
+          end
+        End
+      else
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(*args)
+            [:#{event}, *args]
+          end
+        End
+      end
+    end
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(tok)
+          [:@#{event}, tok, [lineno(), column()]]
+        end
+      End
+    end
+  end
+
+  class SexpBuilder < ::Ripper   #:nodoc:
+    private
+
+    PARSER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(*args)
+          args.unshift :#{event}
+          args
+        end
+      End
+    end
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(tok)
+          [:@#{event}, tok, [lineno(), column()]]
+        end
+      End
+    end
+  end
+
+end