summaryrefslogtreecommitdiff
path: root/jni/ruby/ext/ripper/lib
diff options
context:
space:
mode:
authorJari Vetoniemi <jari.vetoniemi@indooratlas.com>2020-03-16 18:49:26 +0900
committerJari Vetoniemi <jari.vetoniemi@indooratlas.com>2020-03-30 00:39:06 +0900
commitfcbf63e62c627deae76c1b8cb8c0876c536ed811 (patch)
tree64cb17de3f41a2b6fef2368028fbd00349946994 /jni/ruby/ext/ripper/lib
Fresh start
Diffstat (limited to 'jni/ruby/ext/ripper/lib')
-rw-r--r--jni/ruby/ext/ripper/lib/ripper.rb73
-rw-r--r--jni/ruby/ext/ripper/lib/ripper/core.rb70
-rw-r--r--jni/ruby/ext/ripper/lib/ripper/filter.rb77
-rw-r--r--jni/ruby/ext/ripper/lib/ripper/lexer.rb186
-rw-r--r--jni/ruby/ext/ripper/lib/ripper/sexp.rb118
5 files changed, 524 insertions, 0 deletions
diff --git a/jni/ruby/ext/ripper/lib/ripper.rb b/jni/ruby/ext/ripper/lib/ripper.rb
new file mode 100644
index 0000000..542bd40
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper.rb
@@ -0,0 +1,73 @@
+require 'ripper/core'
+require 'ripper/lexer'
+require 'ripper/filter'
+require 'ripper/sexp'
+
+# Ripper is a Ruby script parser.
+#
+# You can get information from the parser with event-based style.
+# Information such as abstract syntax trees or simple lexical analysis of the
+# Ruby program.
+#
+# == Usage
+#
+# Ripper provides an easy interface for parsing your program into a symbolic
+# expression tree (or S-expression).
+#
+# Understanding the output of the parser may come as a challenge, it's
+# recommended you use PP to format the output for legibility.
+#
+# require 'ripper'
+# require 'pp'
+#
+# pp Ripper.sexp('def hello(world) "Hello, #{world}!"; end')
+# #=> [:program,
+# [[:def,
+# [:@ident, "hello", [1, 4]],
+# [:paren,
+# [:params, [[:@ident, "world", [1, 10]]], nil, nil, nil, nil, nil, nil]],
+# [:bodystmt,
+# [[:string_literal,
+# [:string_content,
+# [:@tstring_content, "Hello, ", [1, 18]],
+# [:string_embexpr, [[:var_ref, [:@ident, "world", [1, 27]]]]],
+# [:@tstring_content, "!", [1, 33]]]]],
+# nil,
+# nil,
+# nil]]]]
+#
+# You can see in the example above, the expression starts with +:program+.
+#
+# From here, a method definition at +:def+, followed by the method's identifier
+# <code>:@ident</code>. After the method's identifier comes the parentheses
+# +:paren+ and the method parameters under +:params+.
+#
+# Next is the method body, starting at +:bodystmt+ (+stmt+ meaning statement),
+# which contains the full definition of the method.
+#
+# In our case, we're simply returning a String, so next we have the
+# +:string_literal+ expression.
+#
+# Within our +:string_literal+ you'll notice two <code>@tstring_content</code>,
+# this is the literal part for <code>Hello, </code> and <code>!</code>. Between
+# the two <code>@tstring_content</code> statements is a +:string_embexpr+,
+# where _embexpr_ is an embedded expression. Our expression consists of a local
+# variable, or +var_ref+, with the identifier (<code>@ident</code>) of +world+.
+#
+# == Resources
+#
+# * {Ruby Inside}[http://www.rubyinside.com/using-ripper-to-see-how-ruby-is-parsing-your-code-5270.html]
+#
+# == Requirements
+#
+# * ruby 1.9 (support CVS HEAD only)
+# * bison 1.28 or later (Other yaccs do not work)
+#
+# == License
+#
+# Ruby License.
+#
+# Minero Aoki
+# aamine@loveruby.net
+# http://i.loveruby.net
+class Ripper; end
diff --git a/jni/ruby/ext/ripper/lib/ripper/core.rb b/jni/ruby/ext/ripper/lib/ripper/core.rb
new file mode 100644
index 0000000..008ec53
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper/core.rb
@@ -0,0 +1,70 @@
+#
+# $Id: core.rb 36954 2012-09-12 23:04:41Z zzak $
+#
+# Copyright (c) 2003-2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper.so'
+
+class Ripper
+
+ # Parses the given Ruby program read from +src+.
+ # +src+ must be a String or an IO or a object with a #gets method.
+ def Ripper.parse(src, filename = '(ripper)', lineno = 1)
+ new(src, filename, lineno).parse
+ end
+
+ # This array contains name of parser events.
+ PARSER_EVENTS = PARSER_EVENT_TABLE.keys
+
+ # This array contains name of scanner events.
+ SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
+
+ # This array contains name of all ripper events.
+ EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+ private
+
+ #
+ # Parser Events
+ #
+
+ PARSER_EVENT_TABLE.each do |id, arity|
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
+ def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') })
+ #{arity == 0 ? 'nil' : 'a'}
+ end
+ End
+ end
+
+ # This method is called when weak warning is produced by the parser.
+ # +fmt+ and +args+ is printf style.
+ def warn(fmt, *args)
+ end
+
+ # This method is called when strong warning is produced by the parser.
+ # +fmt+ and +args+ is printf style.
+ def warning(fmt, *args)
+ end
+
+ # This method is called when the parser found syntax error.
+ def compile_error(msg)
+ end
+
+ #
+ # Scanner Events
+ #
+
+ SCANNER_EVENTS.each do |id|
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
+ def on_#{id}(token)
+ token
+ end
+ End
+ end
+
+end
diff --git a/jni/ruby/ext/ripper/lib/ripper/filter.rb b/jni/ruby/ext/ripper/lib/ripper/filter.rb
new file mode 100644
index 0000000..b30b085
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper/filter.rb
@@ -0,0 +1,77 @@
+#
+# $Id: filter.rb 36954 2012-09-12 23:04:41Z zzak $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/lexer'
+
+class Ripper
+
+ # This class handles only scanner events,
+ # which are dispatched in the 'right' order (same with input).
+ class Filter
+
+ # Creates a new Ripper::Filter instance, passes parameters +src+,
+ # +filename+, and +lineno+ to Ripper::Lexer.new
+ #
+ # The lexer is for internal use only.
+ def initialize(src, filename = '-', lineno = 1)
+ @__lexer = Lexer.new(src, filename, lineno)
+ @__line = nil
+ @__col = nil
+ end
+
+ # The file name of the input.
+ def filename
+ @__lexer.filename
+ end
+
+ # The line number of the current token.
+ # This value starts from 1.
+ # This method is valid only in event handlers.
+ def lineno
+ @__line
+ end
+
+ # The column number of the current token.
+ # This value starts from 0.
+ # This method is valid only in event handlers.
+ def column
+ @__col
+ end
+
+ # Starts the parser.
+ # +init+ is a data accumulator and is passed to the next event handler (as
+ # of Enumerable#inject).
+ def parse(init = nil)
+ data = init
+ @__lexer.lex.each do |pos, event, tok|
+ @__line, @__col = *pos
+ data = if respond_to?(event, true)
+ then __send__(event, tok, data)
+ else on_default(event, tok, data)
+ end
+ end
+ data
+ end
+
+ private
+
+ # This method is called when some event handler is undefined.
+ # +event+ is :on_XXX, +token+ is the scanned token, and +data+ is a data
+ # accumulator.
+ #
+ # The return value of this method is passed to the next event handler (as
+ # of Enumerable#inject).
+ def on_default(event, token, data)
+ data
+ end
+
+ end
+
+end
diff --git a/jni/ruby/ext/ripper/lib/ripper/lexer.rb b/jni/ruby/ext/ripper/lib/ripper/lexer.rb
new file mode 100644
index 0000000..38b2087
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper/lexer.rb
@@ -0,0 +1,186 @@
+#
+# $Id: lexer.rb 44878 2014-02-07 12:57:44Z zzak $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/core'
+
+class Ripper
+
+ # Tokenizes the Ruby program and returns an array of strings.
+ #
+ # p Ripper.tokenize("def m(a) nil end")
+ # # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"]
+ #
+ def Ripper.tokenize(src, filename = '-', lineno = 1)
+ Lexer.new(src, filename, lineno).tokenize
+ end
+
+ # Tokenizes the Ruby program and returns an array of an array,
+ # which is formatted like <code>[[lineno, column], type, token]</code>.
+ #
+ # require 'ripper'
+ # require 'pp'
+ #
+ # pp Ripper.lex("def m(a) nil end")
+ # #=> [[[1, 0], :on_kw, "def"],
+ # [[1, 3], :on_sp, " " ],
+ # [[1, 4], :on_ident, "m" ],
+ # [[1, 5], :on_lparen, "(" ],
+ # [[1, 6], :on_ident, "a" ],
+ # [[1, 7], :on_rparen, ")" ],
+ # [[1, 8], :on_sp, " " ],
+ # [[1, 9], :on_kw, "nil"],
+ # [[1, 12], :on_sp, " " ],
+ # [[1, 13], :on_kw, "end"]]
+ #
+ def Ripper.lex(src, filename = '-', lineno = 1)
+ Lexer.new(src, filename, lineno).lex
+ end
+
+ class Lexer < ::Ripper #:nodoc: internal use only
+ def tokenize
+ lex().map {|pos, event, tok| tok }
+ end
+
+ def lex
+ parse().sort_by {|pos, event, tok| pos }
+ end
+
+ def parse
+ @buf = []
+ super
+ @buf
+ end
+
+ private
+
+ SCANNER_EVENTS.each do |event|
+ module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
+ def on_#{event}(tok)
+ @buf.push [[lineno(), column()], :on_#{event}, tok]
+ end
+ End
+ end
+ end
+
+ # [EXPERIMENTAL]
+ # Parses +src+ and return a string which was matched to +pattern+.
+ # +pattern+ should be described as Regexp.
+ #
+ # require 'ripper'
+ #
+ # p Ripper.slice('def m(a) nil end', 'ident') #=> "m"
+ # p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+') #=> "m(a)"
+ # p Ripper.slice("<<EOS\nstring\nEOS",
+ # 'heredoc_beg nl $(tstring_content*) heredoc_end', 1)
+ # #=> "string\n"
+ #
+ def Ripper.slice(src, pattern, n = 0)
+ if m = token_match(src, pattern)
+ then m.string(n)
+ else nil
+ end
+ end
+
+ def Ripper.token_match(src, pattern) #:nodoc:
+ TokenPattern.compile(pattern).match(src)
+ end
+
+ class TokenPattern #:nodoc:
+
+ class Error < ::StandardError # :nodoc:
+ end
+ class CompileError < Error # :nodoc:
+ end
+ class MatchError < Error # :nodoc:
+ end
+
+ class << self
+ alias compile new
+ end
+
+ def initialize(pattern)
+ @source = pattern
+ @re = compile(pattern)
+ end
+
+ def match(str)
+ match_list(::Ripper.lex(str))
+ end
+
+ def match_list(tokens)
+ if m = @re.match(map_tokens(tokens))
+ then MatchData.new(tokens, m)
+ else nil
+ end
+ end
+
+ private
+
+ def compile(pattern)
+ if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern)
+ raise CompileError, "invalid char in pattern: #{m[0].inspect}"
+ end
+ buf = ''
+ pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok|
+ case tok
+ when /\w/
+ buf.concat map_token(tok)
+ when '$('
+ buf.concat '('
+ when '('
+ buf.concat '(?:'
+ when /[?*\[\])\.]/
+ buf.concat tok
+ else
+ raise 'must not happen'
+ end
+ end
+ Regexp.compile(buf)
+ rescue RegexpError => err
+ raise CompileError, err.message
+ end
+
+ def map_tokens(tokens)
+ tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join
+ end
+
+ MAP = {}
+ seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
+ SCANNER_EVENT_TABLE.each do |ev, |
+ raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty?
+ MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift
+ end
+
+ def map_token(tok)
+ MAP[tok] or raise CompileError, "unknown token: #{tok}"
+ end
+
+ class MatchData # :nodoc:
+ def initialize(tokens, match)
+ @tokens = tokens
+ @match = match
+ end
+
+ def string(n = 0)
+ return nil unless @match
+ match(n).join
+ end
+
+ private
+
+ def match(n = 0)
+ return [] unless @match
+ @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str }
+ end
+ end
+
+ end
+
+end
diff --git a/jni/ruby/ext/ripper/lib/ripper/sexp.rb b/jni/ruby/ext/ripper/lib/ripper/sexp.rb
new file mode 100644
index 0000000..514a585
--- /dev/null
+++ b/jni/ruby/ext/ripper/lib/ripper/sexp.rb
@@ -0,0 +1,118 @@
+#
+# $Id: sexp.rb 48144 2014-10-26 03:24:18Z nobu $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/core'
+
+class Ripper
+
+ # [EXPERIMENTAL]
+ # Parses +src+ and create S-exp tree.
+ # Returns more readable tree rather than Ripper.sexp_raw.
+ # This method is mainly for developer use.
+ #
+ # require 'ripper'
+ # require 'pp'
+ #
+ # pp Ripper.sexp("def m(a) nil end")
+ # #=> [:program,
+ # [[:def,
+ # [:@ident, "m", [1, 4]],
+ # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil, nil]],
+ # [:bodystmt, [[:var_ref, [:@kw, "nil", [1, 9]]]], nil, nil, nil]]]]
+ #
+ def Ripper.sexp(src, filename = '-', lineno = 1)
+ builder = SexpBuilderPP.new(src, filename, lineno)
+ sexp = builder.parse
+ sexp unless builder.error?
+ end
+
+ # [EXPERIMENTAL]
+ # Parses +src+ and create S-exp tree.
+ # This method is mainly for developer use.
+ #
+ # require 'ripper'
+ # require 'pp'
+ #
+ # pp Ripper.sexp_raw("def m(a) nil end")
+ # #=> [:program,
+ # [:stmts_add,
+ # [:stmts_new],
+ # [:def,
+ # [:@ident, "m", [1, 4]],
+ # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
+ # [:bodystmt,
+ # [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
+ # nil,
+ # nil,
+ # nil]]]]
+ #
+ def Ripper.sexp_raw(src, filename = '-', lineno = 1)
+ builder = SexpBuilder.new(src, filename, lineno)
+ sexp = builder.parse
+ sexp unless builder.error?
+ end
+
+ class SexpBuilderPP < ::Ripper #:nodoc:
+ private
+
+ PARSER_EVENT_TABLE.each do |event, arity|
+ if /_new\z/ =~ event.to_s and arity == 0
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
+ def on_#{event}
+ []
+ end
+ End
+ elsif /_add\z/ =~ event.to_s
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
+ def on_#{event}(list, item)
+ list.push item
+ list
+ end
+ End
+ else
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
+ def on_#{event}(*args)
+ [:#{event}, *args]
+ end
+ End
+ end
+ end
+
+ SCANNER_EVENTS.each do |event|
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
+ def on_#{event}(tok)
+ [:@#{event}, tok, [lineno(), column()]]
+ end
+ End
+ end
+ end
+
+ class SexpBuilder < ::Ripper #:nodoc:
+ private
+
+ PARSER_EVENTS.each do |event|
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
+ def on_#{event}(*args)
+ args.unshift :#{event}
+ args
+ end
+ End
+ end
+
+ SCANNER_EVENTS.each do |event|
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
+ def on_#{event}(tok)
+ [:@#{event}, tok, [lineno(), column()]]
+ end
+ End
+ end
+ end
+
+end