summaryrefslogtreecommitdiff
path: root/jni/ruby/lib/irb/ruby-token.rb
blob: cb95955cdfeecd8750c449d396ab20a81b370136 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
#
#   irb/ruby-token.rb - ruby tokens
#       $Release Version: 0.9.6$
#       $Revision: 47298 $
#       by Keiju ISHITSUKA(keiju@ruby-lang.org)
#
# --
#
#
#
# :stopdoc:
module RubyToken
  EXPR_BEG = :EXPR_BEG
  EXPR_MID = :EXPR_MID
  EXPR_END = :EXPR_END
  EXPR_ARG = :EXPR_ARG
  EXPR_FNAME = :EXPR_FNAME
  EXPR_DOT = :EXPR_DOT
  EXPR_CLASS = :EXPR_CLASS

  class Token
    def initialize(seek, line_no, char_no)
      @seek = seek
      @line_no = line_no
      @char_no = char_no
    end
    attr_reader :seek, :line_no, :char_no
  end

  class TkNode < Token
    def initialize(seek, line_no, char_no)
      super
    end
    attr_reader :node
  end

  class TkId < Token
    def initialize(seek, line_no, char_no, name)
      super(seek, line_no, char_no)
      @name = name
    end
    attr_reader :name
  end

  class TkVal < Token
    def initialize(seek, line_no, char_no, value = nil)
      super(seek, line_no, char_no)
      @value = value
    end
    attr_reader :value
  end

  class TkOp < Token
    attr_accessor :name
  end

  class TkOPASGN < TkOp
    def initialize(seek, line_no, char_no, op)
      super(seek, line_no, char_no)
      op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
      @op = op
    end
    attr_reader :op
  end

  class TkUnknownChar < Token
    def initialize(seek, line_no, char_no, id)
      super(seek, line_no, char_no)
      @name = name
    end
    attr_reader :name
  end

  class TkError < Token
  end

  def Token(token, value = nil)
    case token
    when String
      if (tk = TkReading2Token[token]).nil?
        IRB.fail TkReading2TokenNoKey, token
      end
      tk = Token(tk[0], value)
      if tk.kind_of?(TkOp)
        tk.name = token
      end
      return tk
    when Symbol
      if (tk = TkSymbol2Token[token]).nil?
        IRB.fail TkSymbol2TokenNoKey, token
      end
      return Token(tk[0], value)
    else
      if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
        token.new(@prev_seek, @prev_line_no, @prev_char_no)
      else
        token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
      end
    end
  end

  TokenDefinitions = [
    [:TkCLASS,      TkId,  "class",  EXPR_CLASS],
    [:TkMODULE,     TkId,  "module", EXPR_BEG],
    [:TkDEF,        TkId,  "def",    EXPR_FNAME],
    [:TkUNDEF,      TkId,  "undef",  EXPR_FNAME],
    [:TkBEGIN,      TkId,  "begin",  EXPR_BEG],
    [:TkRESCUE,     TkId,  "rescue", EXPR_MID],
    [:TkENSURE,     TkId,  "ensure", EXPR_BEG],
    [:TkEND,        TkId,  "end",    EXPR_END],
    [:TkIF,         TkId,  "if",     EXPR_BEG, :TkIF_MOD],
    [:TkUNLESS,     TkId,  "unless", EXPR_BEG, :TkUNLESS_MOD],
    [:TkTHEN,       TkId,  "then",   EXPR_BEG],
    [:TkELSIF,      TkId,  "elsif",  EXPR_BEG],
    [:TkELSE,       TkId,  "else",   EXPR_BEG],
    [:TkCASE,       TkId,  "case",   EXPR_BEG],
    [:TkWHEN,       TkId,  "when",   EXPR_BEG],
    [:TkWHILE,      TkId,  "while",  EXPR_BEG, :TkWHILE_MOD],
    [:TkUNTIL,      TkId,  "until",  EXPR_BEG, :TkUNTIL_MOD],
    [:TkFOR,        TkId,  "for",    EXPR_BEG],
    [:TkBREAK,      TkId,  "break",  EXPR_END],
    [:TkNEXT,       TkId,  "next",   EXPR_END],
    [:TkREDO,       TkId,  "redo",   EXPR_END],
    [:TkRETRY,      TkId,  "retry",  EXPR_END],
    [:TkIN,         TkId,  "in",     EXPR_BEG],
    [:TkDO,         TkId,  "do",     EXPR_BEG],
    [:TkRETURN,     TkId,  "return", EXPR_MID],
    [:TkYIELD,      TkId,  "yield",  EXPR_END],
    [:TkSUPER,      TkId,  "super",  EXPR_END],
    [:TkSELF,       TkId,  "self",   EXPR_END],
    [:TkNIL,        TkId,  "nil",    EXPR_END],
    [:TkTRUE,       TkId,  "true",   EXPR_END],
    [:TkFALSE,      TkId,  "false",  EXPR_END],
    [:TkAND,        TkId,  "and",    EXPR_BEG],
    [:TkOR,         TkId,  "or",     EXPR_BEG],
    [:TkNOT,        TkId,  "not",    EXPR_BEG],
    [:TkIF_MOD,     TkId],
    [:TkUNLESS_MOD, TkId],
    [:TkWHILE_MOD,  TkId],
    [:TkUNTIL_MOD,  TkId],
    [:TkALIAS,      TkId,  "alias",    EXPR_FNAME],
    [:TkDEFINED,    TkId,  "defined?", EXPR_END],
    [:TklBEGIN,     TkId,  "BEGIN",    EXPR_END],
    [:TklEND,       TkId,  "END",      EXPR_END],
    [:Tk__LINE__,   TkId,  "__LINE__", EXPR_END],
    [:Tk__FILE__,   TkId,  "__FILE__", EXPR_END],

    [:TkIDENTIFIER, TkId],
    [:TkFID,        TkId],
    [:TkGVAR,       TkId],
    [:TkCVAR,       TkId],
    [:TkIVAR,       TkId],
    [:TkCONSTANT,   TkId],

    [:TkINTEGER,    TkVal],
    [:TkFLOAT,      TkVal],
    [:TkSTRING,     TkVal],
    [:TkXSTRING,    TkVal],
    [:TkREGEXP,     TkVal],
    [:TkSYMBOL,     TkVal],

    [:TkDSTRING,    TkNode],
    [:TkDXSTRING,   TkNode],
    [:TkDREGEXP,    TkNode],
    [:TkNTH_REF,    TkNode],
    [:TkBACK_REF,   TkNode],

    [:TkUPLUS,      TkOp,   "+@"],
    [:TkUMINUS,     TkOp,   "-@"],
    [:TkPOW,        TkOp,   "**"],
    [:TkCMP,        TkOp,   "<=>"],
    [:TkEQ,         TkOp,   "=="],
    [:TkEQQ,        TkOp,   "==="],
    [:TkNEQ,        TkOp,   "!="],
    [:TkGEQ,        TkOp,   ">="],
    [:TkLEQ,        TkOp,   "<="],
    [:TkANDOP,      TkOp,   "&&"],
    [:TkOROP,       TkOp,   "||"],
    [:TkMATCH,      TkOp,   "=~"],
    [:TkNMATCH,     TkOp,   "!~"],
    [:TkDOT2,       TkOp,   ".."],
    [:TkDOT3,       TkOp,   "..."],
    [:TkAREF,       TkOp,   "[]"],
    [:TkASET,       TkOp,   "[]="],
    [:TkLSHFT,      TkOp,   "<<"],
    [:TkRSHFT,      TkOp,   ">>"],
    [:TkCOLON2,     TkOp],
    [:TkCOLON3,     TkOp],
    [:TkASSOC,      TkOp,   "=>"],
    [:TkQUESTION,   TkOp,   "?"],        #?
    [:TkCOLON,      TkOp,   ":"],        #:

    [:TkfLPAREN],         # func( #
    [:TkfLBRACK],         # func[ #
    [:TkfLBRACE],         # func{ #
    [:TkSTAR],            # *arg
    [:TkAMPER],           # &arg #
    [:TkSYMBEG],          # :SYMBOL

    [:TkGT,         TkOp,   ">"],
    [:TkLT,         TkOp,   "<"],
    [:TkPLUS,       TkOp,   "+"],
    [:TkMINUS,      TkOp,   "-"],
    [:TkMULT,       TkOp,   "*"],
    [:TkDIV,        TkOp,   "/"],
    [:TkMOD,        TkOp,   "%"],
    [:TkBITOR,      TkOp,   "|"],
    [:TkBITXOR,     TkOp,   "^"],
    [:TkBITAND,     TkOp,   "&"],
    [:TkBITNOT,     TkOp,   "~"],
    [:TkNOTOP,      TkOp,   "!"],

    [:TkBACKQUOTE,  TkOp,   "`"],

    [:TkASSIGN,     Token,  "="],
    [:TkDOT,        Token,  "."],
    [:TkLPAREN,     Token,  "("],  #(exp)
    [:TkLBRACK,     Token,  "["],  #[arry]
    [:TkLBRACE,     Token,  "{"],  #{hash}
    [:TkRPAREN,     Token,  ")"],
    [:TkRBRACK,     Token,  "]"],
    [:TkRBRACE,     Token,  "}"],
    [:TkCOMMA,      Token,  ","],
    [:TkSEMICOLON,  Token,  ";"],

    [:TkCOMMENT],
    [:TkRD_COMMENT],
    [:TkSPACE],
    [:TkNL],
    [:TkEND_OF_SCRIPT],

    [:TkBACKSLASH,  TkUnknownChar,  "\\"],
    [:TkAT,         TkUnknownChar,  "@"],
    [:TkDOLLAR,     TkUnknownChar,  "$"],
  ]

  # {reading => token_class}
  # {reading => [token_class, *opt]}
  TkReading2Token = {}
  TkSymbol2Token = {}

  def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
    token_n = token_n.id2name if token_n.kind_of?(Symbol)
    if RubyToken.const_defined?(token_n)
      IRB.fail AlreadyDefinedToken, token_n
    end
    token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")

    if reading
      if TkReading2Token[reading]
        IRB.fail TkReading2TokenDuplicateError, token_n, reading
      end
      if opts.empty?
        TkReading2Token[reading] = [token_c]
      else
        TkReading2Token[reading] = [token_c].concat(opts)
      end
    end
    TkSymbol2Token[token_n.intern] = token_c
  end

  for defs in TokenDefinitions
    def_token(*defs)
  end
end
# :startdoc: