# coding: US-ASCII
#--
# irb/ruby-lex.rb - ruby lexcal analyzer
# $Release Version: 0.9.5$
# $Revision: 17979 $
# $Date: 2008-07-09 10:17:05 -0700 (Wed, 09 Jul 2008) $
# by Keiju ISHITSUKA(keiju@ruby-lang.org)
#
#++
require "e2mmap"
require "irb/slex"
require "stringio"
##
# Ruby lexer adapted from irb.
#
# The internals are not documented because they are scary.
class RDoc::RubyLex
##
# Raised upon invalid input
class Error < RDoc::Error
end
# :stopdoc:
extend Exception2MessageMapper
def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
def_exception(:TkReading2TokenDuplicateError,
"key duplicate(token_n='%s', key='%s')")
def_exception(:SyntaxError, "%s")
def_exception(:TerminateLineInput, "Terminate Line Input")
include RDoc::RubyToken
include IRB
attr_accessor :continue
attr_accessor :lex_state
attr_reader :reader
class << self
attr_accessor :debug_level
end
def self.debug?
@debug_level > 0
end
self.debug_level = 0
# :startdoc:
##
# Returns an Array of +ruby+ tokens. See ::new for a description of
# +options+.
def self.tokenize ruby, options
tokens = []
scanner = RDoc::RubyLex.new ruby, options
scanner.exception_on_syntax_error = true
while token = scanner.token do
tokens << token
end
tokens
end
##
# Creates a new lexer for +content+. +options+ is an RDoc::Options, only
# +tab_width is used.
def initialize(content, options)
lex_init
if /\t/ =~ content then
tab_width = options.tab_width
content = content.split(/\n/).map do |line|
1 while line.gsub!(/\t+/) {
' ' * (tab_width*$&.length - $`.length % tab_width)
} && $~
line
end.join("\n")
end
content << "\n" unless content[-1, 1] == "\n"
set_input StringIO.new content
@base_char_no = 0
@char_no = 0
@exp_line_no = @line_no = 1
@here_readed = []
@readed = []
@rests = []
@seek = 0
@here_header = false
@indent = 0
@indent_stack = []
@lex_state = :EXPR_BEG
@space_seen = false
@continue = false
@line = ""
@skip_space = false
@readed_auto_clean_up = false
@exception_on_syntax_error = true
@prompt = nil
@prev_seek = nil
@ltype = nil
end
# :stopdoc:
def inspect # :nodoc:
"#<%s:0x%x pos %d lex_state %p space_seen %p>" % [
self.class, object_id,
@io.pos, @lex_state, @space_seen,
]
end
attr_accessor :skip_space
attr_accessor :readed_auto_clean_up
attr_accessor :exception_on_syntax_error
attr_reader :seek
attr_reader :char_no
attr_reader :line_no
attr_reader :indent
| 