Module: MilkTea::Lexer::FormatStrings

Included in:
MilkTea::Lexer
Defined in:
lib/milk_tea/core/lexer/format_strings.rb

Overview

Format string lexing (f"...#{expr}...") and the interpolation scanning / format-spec splitting shared with format heredocs.

Instance Method Summary collapse

Instance Method Details

#advance_heredoc_position(char:, line:, column:, base_column:) ⇒ Object 



50
51
52
53
54
55
56
 
# File 'lib/milk_tea/core/lexer/format_strings.rb', line 50

def advance_heredoc_position(char:, line:, column:, base_column:)
  if char == "\n"
    [line + 1, base_column]
  else
    [line, column + 1]
  end
end

#format_spec_suffix?(source) ⇒ Boolean

Returns:

  • (Boolean) 


218
219
220
 
# File 'lib/milk_tea/core/lexer/format_strings.rb', line 218

def format_spec_suffix?(source)
  source && source.strip.match?(/\A(?:\.\d+|[xXoObB])\z/)
end

#lex_format_string(line, index, line_number, line_offset:) ⇒ Object

Raises:



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/milk_tea/core/lexer/format_strings.rb', line 58

def lex_format_string(line, index, line_number, line_offset:)
  start = index
  index += 2
  text = +""
  parts = []

  while index < line.length
    char = line[index]
    if char == '"'
      parts << { kind: :text, value: text } unless text.empty?
      lexeme = line[start..index]
      @tokens << token(:fstring, lexeme, parts, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 1)
      return index + 1
    end

    if char == "#" && line[index + 1] == "{"
      parts << { kind: :text, value: text } unless text.empty?
      text = +""
      expr_start = index + 2
      expr_end = scan_format_interpolation_end(line, expr_start, line_number, start + 1, recover: @recovery_errors)
      raw_source = line[expr_start...expr_end]
      if raw_source.strip.empty?
        raise LexError.new("empty format interpolation", line: line_number, column: index + 1, path: @path)
      end

      source, format_spec = split_format_interpolation_source(raw_source)
      parts << { kind: :expr, source:, format_spec:, line: line_number, column: expr_start + 1 }
      index = expr_end + 1
      next
    end

    if char == "\\"
      next_char = line[index + 1]
      raise LexError.new("unterminated format string literal", line: line_number, column: start + 1, path: @path) unless next_char

      text << decode_escape(next_char)
      index += 2
      next
    end

    text << char
    index += 1
  end

  if @recovery_errors
    @recovery_errors << LexError.new("unterminated format string literal", line: line_number, column: start + 1, path: @path)
    parts << { kind: :text, value: text } unless text.empty?
    lexeme = line[start..]
    @tokens << token(:fstring, lexeme, parts, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + line.length)
    return line.length
  end

  raise LexError.new("unterminated format string literal", line: line_number, column: start + 1, path: @path)
end

#parse_format_heredoc_parts(content, start_line:, start_column:) ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/milk_tea/core/lexer/format_strings.rb', line 8

def parse_format_heredoc_parts(content, start_line:, start_column:)
  parts = []
  text = +""
  index = 0
  line = start_line
  column = start_column
  base_column = start_column

  while index < content.length
    char = content[index]
    if char == "#" && content[index + 1] == "{"
      parts << { kind: :text, value: text } unless text.empty?
      text = +""

      expr_start = index + 2
      expr_line = line
      expr_column = column + 2
      expr_end = scan_format_interpolation_end(content, expr_start, expr_line, expr_column, recover: @recovery_errors)
      raw_source = content[expr_start...expr_end]
      if raw_source.strip.empty?
        raise LexError.new("empty format interpolation", line: line, column:, path: @path)
      end

      source, format_spec = split_format_interpolation_source(raw_source)
      parts << { kind: :expr, source:, format_spec:, line: expr_line, column: expr_column }

      while index <= expr_end
        line, column = advance_heredoc_position(char: content[index], line:, column:, base_column:)
        index += 1
      end
      next
    end

    text << char
    line, column = advance_heredoc_position(char:, line:, column:, base_column:)
    index += 1
  end

  parts << { kind: :text, value: text } unless text.empty?
  parts
end

#scan_format_interpolation_end(line, index, line_number, column, recover: false) ⇒ Object

Raises:



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/milk_tea/core/lexer/format_strings.rb', line 113

def scan_format_interpolation_end(line, index, line_number, column, recover: false)
  depth = 1

  while index < line.length
    char = line[index]

    if char == '"'
      index = skip_string_contents(line, index, line_number)
      next
    end

    if char == "{"
      depth += 1
      index += 1
      next
    end

    if char == "}"
      depth -= 1
      return index if depth == 0

      index += 1
      next
    end

    index += 1
  end

  if recover
    @recovery_errors << LexError.new("unterminated format interpolation", line: line_number, column:, path: @path) if @recovery_errors
    return line.length
  end

  raise LexError.new("unterminated format interpolation", line: line_number, column:, path: @path)
end

#skip_interpolation_string_contents(source, index) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/milk_tea/core/lexer/format_strings.rb', line 200

def skip_interpolation_string_contents(source, index)
  index += 1

  while index < source.length
    char = source[index]
    return index + 1 if char == '"'

    if char == "\\"
      index += 2
      next
    end

    index += 1
  end

  source.length
end

#skip_string_contents(line, index, line_number) ⇒ Object

Raises:



149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/milk_tea/core/lexer/format_strings.rb', line 149

def skip_string_contents(line, index, line_number)
  index += 1

  while index < line.length
    char = line[index]
    return index + 1 if char == '"'

    if char == "\\"
      raise LexError.new("unterminated string literal", line: line_number, column: index + 1, path: @path) unless line[index + 1]

      index += 2
      next
    end

    index += 1
  end

  raise LexError.new("unterminated string literal", line: line_number, column: index + 1, path: @path)
end

#split_format_interpolation_source(source) ⇒ Object 



169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/milk_tea/core/lexer/format_strings.rb', line 169

def split_format_interpolation_source(source)
  depth = 0
  format_spec_index = nil
  index = 0

  while index < source.length
    char = source[index]

    if char == '"'
      index = skip_interpolation_string_contents(source, index)
      next
    end

    case char
      when "(", "[", "{"
        depth += 1
      when ")", "]", "}"
        depth -= 1 if depth.positive?
      when ":"
        suffix = source[(index + 1)..]
        format_spec_index = index if depth.zero? && format_spec_suffix?(suffix)
    end

    index += 1
  end

  return [source, nil] unless format_spec_index

  [source[0...format_spec_index], source[(format_spec_index + 1)..]]
end