Class: Foxtail::Syntax::Parser::Stream

Inherits:
Object
  • Object
show all
Defined in:
lib/foxtail/syntax/parser/stream.rb

Overview

Ruby equivalent of fluent.js FluentParserStream Handles character stream processing with CRLF normalization and parsing utilities

Constant Summary collapse

EOL =

Constants

"\n"
EOF =

End of file marker

nil
SPECIAL_LINE_START_CHARS =

Characters that have special meaning at the start of a line in FTL

["}", ".", "[", "*"].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(string) ⇒ Stream

Returns a new instance of Stream.



25
26
27
28
29
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 25

def initialize(string)
  @string = string
  @index = 0
  @peek_offset = 0
end

Instance Attribute Details

#index ⇒ Object (readonly)

Returns the value of attribute index.



22
23
24
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 22

def index
  @index
end

#peek_offset ⇒ Object (readonly)

Returns the value of attribute peek_offset.



23
24
25
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 23

def peek_offset
  @peek_offset
end

#string ⇒ Object (readonly)

Returns the value of attribute string.



21
22
23
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 21

def string
  @string
end

Instance Method Details

#attribute_start? ⇒ Boolean

Returns:

  • (Boolean) 


271
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 271

def attribute_start? = current_peek == "."

#char_at(offset) ⇒ Object

Base stream methods (from ParserStream)



33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 33

def char_at(offset)
  # When the cursor is at CRLF, return LF but don't move the cursor.
  # The cursor still points to the EOL position, which in this case is the
  # beginning of the compound CRLF sequence. This ensures slices of
  # [inclusive, exclusive) continue to work properly.
  if @string[offset] == "\r" && @string[offset + 1] == "\n"
    return "\n"
  end

  @string[offset]
end

#char_id_start?(ch) ⇒ Boolean

Returns:

  • (Boolean) 


173
174
175
176
177
178
179
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 173

def char_id_start?(ch)
  return false if ch == EOF

  cc = ch.ord
  cc.between?(97, 122) || # a-z
    cc.between?(65, 90) # A-Z
end

#char_pattern_continuation?(ch) ⇒ Boolean

Returns:

  • (Boolean) 


197
198
199
200
201
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 197

def char_pattern_continuation?(ch)
  return false if ch == EOF

  !SPECIAL_LINE_START_CHARS.include?(ch)
end

#current_char ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



46
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 46

def current_char = char_at(@index)

#current_peek ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



49
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 49

def current_peek = char_at(@index + @peek_offset)

#expect_char(ch) ⇒ Object

Raises:



135
136
137
138
139
140
141
142
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 135

def expect_char(ch)
  if current_char == ch
    self.next
    return
  end

  raise ParseError.new("E0003", ch)
end

#expect_line_end ⇒ Object

Raises:



144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 144

def expect_line_end
  if current_char == EOF
    # EOF is a valid line end in Fluent.
    return
  end

  if current_char == EOL
    self.next
    return
  end

  # Unicode Character 'SYMBOL FOR NEWLINE' (U+2424)
  raise ParseError.new("E0003", "\u2424")
end

#identifier_start? ⇒ Boolean

Returns:

  • (Boolean) 


181
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 181

def identifier_start? = char_id_start?(current_peek)

#next ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



52
53
54
55
56
57
58
59
60
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 52

def next
  @peek_offset = 0
  # Skip over the CRLF as if it was a single character.
  if @string[@index] == "\r" && @string[@index + 1] == "\n"
    @index += 1
  end
  @index += 1
  @string[@index]
end

#next_line_comment?(level = -1)) ⇒ Boolean

Parameters:

  • level (defaults to: -1)) —

    • -1: any, 0: comment, 1: group comment, 2: resource comment

Returns:

  • (Boolean) 


231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 231

def next_line_comment?(level=-1)
  return false if current_char != EOL

  i = 0

  while i <= level || (level == -1 && i < 3)
    if peek != "#"
      if i <= level && level != -1
        reset_peek
        return false
      end
      break
    end
    i += 1
  end

  # The first char after #, ## or ###.
  ch = peek
  if ch == " " || ch == EOL
    reset_peek
    return true
  end

  reset_peek
  false
end

#number_start? ⇒ Boolean

Returns:

  • (Boolean) 


183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 183

def number_start?
  ch = current_char == "-" ? peek : current_char

  if ch == EOF
    reset_peek
    return false
  end

  cc = ch.ord
  is_digit = cc.between?(48, 57) # 0-9
  reset_peek
  is_digit
end

#peek ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



63
64
65
66
67
68
69
70
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 63

def peek
  # Skip over the CRLF as if it was a single character.
  if @string[@index + @peek_offset] == "\r" && @string[@index + @peek_offset + 1] == "\n"
    @peek_offset += 1
  end
  @peek_offset += 1
  @string[@index + @peek_offset]
end

#peek_blank ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



127
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 127

def peek_blank = (peek while current_peek == " " || current_peek == EOL)

#peek_blank_block ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 98

def peek_blank_block
  blank = ""
  loop do
    line_start = @peek_offset
    peek_blank_inline
    if current_peek == EOL
      blank += EOL
      peek
      next
    end
    if current_peek == EOF
      # Treat the blank line at EOF as a blank block.
      return blank
    end

    # Any other char; reset to column 1 on this line.
    reset_peek(line_start)
    return blank
  end
end

#peek_blank_inline ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



84
85
86
87
88
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 84

def peek_blank_inline
  start = @index + @peek_offset
  peek while current_peek == " "
  @string.slice(start, @index + @peek_offset - start)
end

#reset_peek(offset = 0) ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



73
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 73

def reset_peek(offset=0) = @peek_offset = offset

#skip_blank ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



130
131
132
133
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 130

def skip_blank
  peek_blank
  skip_to_peek
end

#skip_blank_block ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



120
121
122
123
124
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 120

def skip_blank_block
  blank = peek_blank_block
  skip_to_peek
  blank
end

#skip_blank_inline ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



91
92
93
94
95
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 91

def skip_blank_inline
  blank = peek_blank_inline
  skip_to_peek
  blank
end

#skip_to_next_entry_start(junk_start) ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 274

def skip_to_next_entry_start(junk_start)
  last_newline = @string.rindex(EOL, @index)
  if last_newline && junk_start < last_newline
    # Last seen newline is _after_ the junk start. It's safe to rewind
    # without the risk of resuming at the same broken entry.
    @index = last_newline
  end
  while current_char
    # We're only interested in beginnings of line.
    unless current_char == EOL
      self.next
      next
    end

    # Break if the first char in this line looks like an entry start.
    first = self.next
    if char_id_start?(first) || first == "-" || first == "#"
      break
    end
  end
end

#skip_to_peek ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



76
77
78
79
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 76

def skip_to_peek
  @index += @peek_offset
  @peek_offset = 0
end

#take_char ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 160

def take_char
  ch = current_char
  if ch == EOF
    return EOF
  end

  if yield(ch)
    self.next
    return ch
  end
  nil
end

#take_digit ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



319
320
321
322
323
324
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 319

def take_digit
  take_char do |ch|
    cc = ch.ord
    cc.between?(48, 57) # 0-9
  end
end

#take_hex_digit ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



327
328
329
330
331
332
333
334
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 327

def take_hex_digit
  take_char do |ch|
    cc = ch.ord
    cc.between?(48, 57) || # 0-9
      cc.between?(65, 70) ||  # A-F
      cc.between?(97, 102)    # a-f
  end
end

#take_id_char ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



307
308
309
310
311
312
313
314
315
316
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 307

def take_id_char
  take_char do |ch|
    cc = ch.ord
    cc.between?(97, 122) || # a-z
      cc.between?(65, 90) ||  # A-Z
      cc.between?(48, 57) ||  # 0-9
      cc == 95 ||                # _
      cc == 45                   # -
  end
end

#take_id_start ⇒ Object

Raises:



296
297
298
299
300
301
302
303
304
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 296

def take_id_start
  if char_id_start?(current_char)
    ret = current_char
    self.next
    return ret
  end

  raise ParseError.new("E0004", "a-zA-Z")
end

#value_continuation? ⇒ Boolean

Returns:

  • (Boolean) 


209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 209

def value_continuation?
  column1 = @peek_offset
  peek_blank_inline

  if current_peek == "{"
    reset_peek(column1)
    return true
  end

  if @peek_offset - column1 == 0
    return false
  end

  if char_pattern_continuation?(current_peek)
    reset_peek(column1)
    return true
  end

  false
end

#value_start? ⇒ Boolean

Returns:

  • (Boolean) 


203
204
205
206
207
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 203

def value_start?
  # Inline Patterns may start with any char.
  ch = current_peek
  ch != EOL && ch != EOF
end

#variant_start? ⇒ Boolean

Returns:

  • (Boolean) 


258
259
260
261
262
263
264
265
266
267
268
269
 
# File 'lib/foxtail/syntax/parser/stream.rb', line 258

def variant_start?
  current_peek_offset = @peek_offset
  if current_peek == "*"
    peek
  end
  if current_peek == "["
    reset_peek(current_peek_offset)
    return true
  end
  reset_peek(current_peek_offset)
  false
end