summaryrefslogtreecommitdiff
path: root/jni/ruby/lib/rexml/text.rb
blob: d3242ee46db0ae58a2e76a3680136ff3a9812882 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
require 'rexml/security'
require 'rexml/entity'
require 'rexml/doctype'
require 'rexml/child'
require 'rexml/doctype'
require 'rexml/parseexception'

module REXML
  # Represents text nodes in an XML document
  class Text < Child
    include Comparable
    # The order in which the substitutions occur
    SPECIALS = [ /&(?!#?[\w-]+;)/u, /</u, />/u, /"/u, /'/u, /\r/u ]
    SUBSTITUTES = ['&amp;', '&lt;', '&gt;', '&quot;', '&apos;', '&#13;']
    # Characters which are substituted in written strings
    SLAICEPS = [ '<', '>', '"', "'", '&' ]
    SETUTITSBUS = [ /&lt;/u, /&gt;/u, /&quot;/u, /&apos;/u, /&amp;/u ]

    # If +raw+ is true, then REXML leaves the value alone
    attr_accessor :raw

    NEEDS_A_SECOND_CHECK = /(<|&((#{Entity::NAME});|(#0*((?:\d+)|(?:x[a-fA-F0-9]+)));)?)/um
    NUMERICENTITY = /&#0*((?:\d+)|(?:x[a-fA-F0-9]+));/
    VALID_CHAR = [
      0x9, 0xA, 0xD,
      (0x20..0xD7FF),
      (0xE000..0xFFFD),
      (0x10000..0x10FFFF)
    ]

    if String.method_defined? :encode
      VALID_XML_CHARS = Regexp.new('^['+
        VALID_CHAR.map { |item|
          case item
          when Fixnum
            [item].pack('U').force_encoding('utf-8')
          when Range
            [item.first, '-'.ord, item.last].pack('UUU').force_encoding('utf-8')
          end
        }.join +
      ']*$')
    else
      VALID_XML_CHARS = /^(
           [\x09\x0A\x0D\x20-\x7E]            # ASCII
         | [\xC2-\xDF][\x80-\xBF]             # non-overlong 2-byte
         |  \xE0[\xA0-\xBF][\x80-\xBF]        # excluding overlongs
         | [\xE1-\xEC\xEE][\x80-\xBF]{2}      # straight 3-byte
         |  \xEF[\x80-\xBE]{2}                #
         |  \xEF\xBF[\x80-\xBD]               # excluding U+fffe and U+ffff
         |  \xED[\x80-\x9F][\x80-\xBF]        # excluding surrogates
         |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
         | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
         |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
       )*$/nx;
    end

    # Constructor
    # +arg+ if a String, the content is set to the String.  If a Text,
    # the object is shallowly cloned.
    #
    # +respect_whitespace+ (boolean, false) if true, whitespace is
    # respected
    #
    # +parent+ (nil) if this is a Parent object, the parent
    # will be set to this.
    #
    # +raw+ (nil) This argument can be given three values.
    # If true, then the value of used to construct this object is expected to
    # contain no unescaped XML markup, and REXML will not change the text. If
    # this value is false, the string may contain any characters, and REXML will
    # escape any and all defined entities whose values are contained in the
    # text.  If this value is nil (the default), then the raw value of the
    # parent will be used as the raw value for this node.  If there is no raw
    # value for the parent, and no value is supplied, the default is false.
    # Use this field if you have entities defined for some text, and you don't
    # want REXML to escape that text in output.
    #   Text.new( "<&", false, nil, false ) #-> "&lt;&amp;"
    #   Text.new( "&lt;&amp;", false, nil, false ) #-> "&amp;lt;&amp;amp;"
    #   Text.new( "<&", false, nil, true )  #-> Parse exception
    #   Text.new( "&lt;&amp;", false, nil, true )  #-> "&lt;&amp;"
    #   # Assume that the entity "s" is defined to be "sean"
    #   # and that the entity    "r" is defined to be "russell"
    #   Text.new( "sean russell" )          #-> "&s; &r;"
    #   Text.new( "sean russell", false, nil, true ) #-> "sean russell"
    #
    # +entity_filter+ (nil) This can be an array of entities to match in the
    # supplied text.  This argument is only useful if +raw+ is set to false.
    #   Text.new( "sean russell", false, nil, false, ["s"] ) #-> "&s; russell"
    #   Text.new( "sean russell", false, nil, true, ["s"] ) #-> "sean russell"
    # In the last example, the +entity_filter+ argument is ignored.
    #
    # +illegal+ INTERNAL USE ONLY
    def initialize(arg, respect_whitespace=false, parent=nil, raw=nil,
      entity_filter=nil, illegal=NEEDS_A_SECOND_CHECK )

      @raw = false
      @parent = nil

      if parent
        super( parent )
        @raw = parent.raw
      end

      @raw = raw unless raw.nil?
      @entity_filter = entity_filter
      clear_cache

      if arg.kind_of? String
        @string = arg.dup
        @string.squeeze!(" \n\t") unless respect_whitespace
      elsif arg.kind_of? Text
        @string = arg.to_s
        @raw = arg.raw
      elsif
        raise "Illegal argument of type #{arg.type} for Text constructor (#{arg})"
      end

      @string.gsub!( /\r\n?/, "\n" )

      Text.check(@string, illegal, doctype) if @raw
    end

    def parent= parent
      super(parent)
      Text.check(@string, NEEDS_A_SECOND_CHECK, doctype) if @raw and @parent
    end

    # check for illegal characters
    def Text.check string, pattern, doctype

      # illegal anywhere
      if string !~ VALID_XML_CHARS
        if String.method_defined? :encode
          string.chars.each do |c|
            case c.ord
            when *VALID_CHAR
            else
              raise "Illegal character #{c.inspect} in raw string \"#{string}\""
            end
          end
        else
          string.scan(/[\x00-\x7F]|[\x80-\xBF][\xC0-\xF0]*|[\xC0-\xF0]/n) do |c|
            case c.unpack('U')
            when *VALID_CHAR
            else
              raise "Illegal character #{c.inspect} in raw string \"#{string}\""
            end
          end
        end
      end

      # context sensitive
      string.scan(pattern) do
        if $1[-1] != ?;
          raise "Illegal character '#{$1}' in raw string \"#{string}\""
        elsif $1[0] == ?&
          if $5 and $5[0] == ?#
            case ($5[1] == ?x ? $5[2..-1].to_i(16) : $5[1..-1].to_i)
            when *VALID_CHAR
            else
              raise "Illegal character '#{$1}' in raw string \"#{string}\""
            end
          # FIXME: below can't work but this needs API change.
          # elsif @parent and $3 and !SUBSTITUTES.include?($1)
          #   if !doctype or !doctype.entities.has_key?($3)
          #     raise "Undeclared entity '#{$1}' in raw string \"#{string}\""
          #   end
          end
        end
      end
    end

    def node_type
      :text
    end

    def empty?
      @string.size==0
    end


    def clone
      return Text.new(self)
    end


    # Appends text to this text node.  The text is appended in the +raw+ mode
    # of this text node.
    #
    # +returns+ the text itself to enable method chain like
    # 'text << "XXX" << "YYY"'.
    def <<( to_append )
      @string << to_append.gsub( /\r\n?/, "\n" )
      clear_cache
      self
    end


    # +other+ a String or a Text
    # +returns+ the result of (to_s <=> arg.to_s)
    def <=>( other )
      to_s() <=> other.to_s
    end

    def doctype
      if @parent
        doc = @parent.document
        doc.doctype if doc
      end
    end

    REFERENCE = /#{Entity::REFERENCE}/
    # Returns the string value of this text node.  This string is always
    # escaped, meaning that it is a valid XML text node string, and all
    # entities that can be escaped, have been inserted.  This method respects
    # the entity filter set in the constructor.
    #
    #   # Assume that the entity "s" is defined to be "sean", and that the
    #   # entity "r" is defined to be "russell"
    #   t = Text.new( "< & sean russell", false, nil, false, ['s'] )
    #   t.to_s   #-> "&lt; &amp; &s; russell"
    #   t = Text.new( "< & &s; russell", false, nil, false )
    #   t.to_s   #-> "&lt; &amp; &s; russell"
    #   u = Text.new( "sean russell", false, nil, true )
    #   u.to_s   #-> "sean russell"
    def to_s
      return @string if @raw
      return @normalized if @normalized

      @normalized = Text::normalize( @string, doctype, @entity_filter )
    end

    def inspect
      @string.inspect
    end

    # Returns the string value of this text.  This is the text without
    # entities, as it might be used programmatically, or printed to the
    # console.  This ignores the 'raw' attribute setting, and any
    # entity_filter.
    #
    #   # Assume that the entity "s" is defined to be "sean", and that the
    #   # entity "r" is defined to be "russell"
    #   t = Text.new( "< & sean russell", false, nil, false, ['s'] )
    #   t.value   #-> "< & sean russell"
    #   t = Text.new( "< & &s; russell", false, nil, false )
    #   t.value   #-> "< & sean russell"
    #   u = Text.new( "sean russell", false, nil, true )
    #   u.value   #-> "sean russell"
    def value
      return @unnormalized if @unnormalized
      @unnormalized = Text::unnormalize( @string, doctype )
    end

    # Sets the contents of this text node.  This expects the text to be
    # unnormalized.  It returns self.
    #
    #   e = Element.new( "a" )
    #   e.add_text( "foo" )   # <a>foo</a>
    #   e[0].value = "bar"    # <a>bar</a>
    #   e[0].value = "<a>"    # <a>&lt;a&gt;</a>
    def value=( val )
      @string = val.gsub( /\r\n?/, "\n" )
      clear_cache
      @raw = false
    end

     def wrap(string, width, addnewline=false)
       # Recursively wrap string at width.
       return string if string.length <= width
       place = string.rindex(' ', width) # Position in string with last ' ' before cutoff
       if addnewline then
         return "\n" + string[0,place] + "\n" + wrap(string[place+1..-1], width)
       else
         return string[0,place] + "\n" + wrap(string[place+1..-1], width)
       end
     end

    def indent_text(string, level=1, style="\t", indentfirstline=true)
      return string if level < 0
      new_string = ''
      string.each_line { |line|
        indent_string = style * level
        new_line = (indent_string + line).sub(/[\s]+$/,'')
        new_string << new_line
      }
      new_string.strip! unless indentfirstline
      return new_string
    end

    # == DEPRECATED
    # See REXML::Formatters
    #
    def write( writer, indent=-1, transitive=false, ie_hack=false )
      Kernel.warn("#{self.class.name}.write is deprecated.  See REXML::Formatters")
      formatter = if indent > -1
          REXML::Formatters::Pretty.new( indent )
        else
          REXML::Formatters::Default.new
        end
      formatter.write( self, writer )
    end

    # FIXME
    # This probably won't work properly
    def xpath
      path = @parent.xpath
      path += "/text()"
      return path
    end

    # Writes out text, substituting special characters beforehand.
    # +out+ A String, IO, or any other object supporting <<( String )
    # +input+ the text to substitute and the write out
    #
    #   z=utf8.unpack("U*")
    #   ascOut=""
    #   z.each{|r|
    #     if r <  0x100
    #       ascOut.concat(r.chr)
    #     else
    #       ascOut.concat(sprintf("&#x%x;", r))
    #     end
    #   }
    #   puts ascOut
    def write_with_substitution out, input
      copy = input.clone
      # Doing it like this rather than in a loop improves the speed
      copy.gsub!( SPECIALS[0], SUBSTITUTES[0] )
      copy.gsub!( SPECIALS[1], SUBSTITUTES[1] )
      copy.gsub!( SPECIALS[2], SUBSTITUTES[2] )
      copy.gsub!( SPECIALS[3], SUBSTITUTES[3] )
      copy.gsub!( SPECIALS[4], SUBSTITUTES[4] )
      copy.gsub!( SPECIALS[5], SUBSTITUTES[5] )
      out << copy
    end

    private
    def clear_cache
      @normalized = nil
      @unnormalized = nil
    end

    # Reads text, substituting entities
    def Text::read_with_substitution( input, illegal=nil )
      copy = input.clone

      if copy =~ illegal
        raise ParseException.new( "malformed text: Illegal character #$& in \"#{copy}\"" )
      end if illegal

      copy.gsub!( /\r\n?/, "\n" )
      if copy.include? ?&
        copy.gsub!( SETUTITSBUS[0], SLAICEPS[0] )
        copy.gsub!( SETUTITSBUS[1], SLAICEPS[1] )
        copy.gsub!( SETUTITSBUS[2], SLAICEPS[2] )
        copy.gsub!( SETUTITSBUS[3], SLAICEPS[3] )
        copy.gsub!( SETUTITSBUS[4], SLAICEPS[4] )
        copy.gsub!( /&#0*((?:\d+)|(?:x[a-f0-9]+));/ ) {
          m=$1
          #m='0' if m==''
          m = "0#{m}" if m[0] == ?x
          [Integer(m)].pack('U*')
        }
      end
      copy
    end

    EREFERENCE = /&(?!#{Entity::NAME};)/
    # Escapes all possible entities
    def Text::normalize( input, doctype=nil, entity_filter=nil )
      copy = input.to_s
      # Doing it like this rather than in a loop improves the speed
      #copy = copy.gsub( EREFERENCE, '&amp;' )
      copy = copy.gsub( "&", "&amp;" )
      if doctype
        # Replace all ampersands that aren't part of an entity
        doctype.entities.each_value do |entity|
          copy = copy.gsub( entity.value,
            "&#{entity.name};" ) if entity.value and
              not( entity_filter and entity_filter.include?(entity.name) )
        end
      else
        # Replace all ampersands that aren't part of an entity
        DocType::DEFAULT_ENTITIES.each_value do |entity|
          copy = copy.gsub(entity.value, "&#{entity.name};" )
        end
      end
      copy
    end

    # Unescapes all possible entities
    def Text::unnormalize( string, doctype=nil, filter=nil, illegal=nil )
      sum = 0
      string.gsub( /\r\n?/, "\n" ).gsub( REFERENCE ) {
        s = Text.expand($&, doctype, filter)
        if sum + s.bytesize > Security.entity_expansion_text_limit
          raise "entity expansion has grown too large"
        else
          sum += s.bytesize
        end
        s
      }
    end

    def Text.expand(ref, doctype, filter)
      if ref[1] == ?#
        if ref[2] == ?x
          [ref[3...-1].to_i(16)].pack('U*')
        else
          [ref[2...-1].to_i].pack('U*')
        end
      elsif ref == '&amp;'
        '&'
      elsif filter and filter.include?( ref[1...-1] )
        ref
      elsif doctype
        doctype.entity( ref[1...-1] ) or ref
      else
        entity_value = DocType::DEFAULT_ENTITIES[ ref[1...-1] ]
        entity_value ? entity_value.value : ref
      end
    end
  end
end