Module: MilkTea::Lexer::Strings

Included in:
MilkTea::Lexer
Defined in:
lib/milk_tea/core/lexer/strings.rb

Overview

String literal lexing (including adjacent-literal continuation), character literal lexing, and escape-sequence decoding.

Instance Method Summary collapse

Instance Method Details

#decode_escape(char) ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/milk_tea/core/lexer/strings.rb', line 151

def decode_escape(char)
  case char
  when "n" then "\n"
  when "r" then "\r"
  when "t" then "\t"
  when "0" then "\0"
  when '"' then '"'
  when "'" then "'"
  when "\\" then "\\"
  else char
  end
end

#lex_char_literal(line, index, line_number, line_offset:) ⇒ Object 



111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/milk_tea/core/lexer/strings.rb', line 111

def lex_char_literal(line, index, line_number, line_offset:)
  start = index
  index += 1
  if index >= line.length
    raise LexError.new("unterminated character literal", line: line_number, column: start + 1, path: @path)
  end

  char = line[index]
  if char == "\\"
    index += 1
    if index >= line.length
      raise LexError.new("unterminated escape in character literal", line: line_number, column: start + 1, path: @path)
    end
    escape_char = line[index]
    if escape_char == "x"
      hex = line[index + 1, 2]
      unless hex&.match?(/\A[0-9a-fA-F]{2}\z/)
        raise LexError.new("invalid hex escape in character literal", line: line_number, column: index + 1, path: @path)
      end
      value = hex.to_i(16)
      index += 3
    else
      value = decode_escape(escape_char).ord
      index += 1
    end
  else
    value = char.ord
    index += 1
  end

  if index >= line.length || line[index] != "'"
    raise LexError.new("expected closing ' in character literal", line: line_number, column: index + 1, path: @path)
  end
  index += 1

  lexeme = line[start...index]
  @tokens << token(:char_literal, lexeme, value, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
  index
end

#lex_string(lines, line_index, line, index, line_number, line_offset:, cstring: false) ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/milk_tea/core/lexer/strings.rb', line 8

def lex_string(lines, line_index, line, index, line_number, line_offset:, cstring: false)
  segment = scan_string_segment(line, index, line_number, cstring:, recover: @recovery_errors)
  consumed_lines = 1
  value = +segment.value
  last_line = line
  last_line_number = line_number
  last_line_offset = line_offset
  last_segment_end = segment.next_index
  last_line_has_newline = lines.fetch(line_index).end_with?("\n")
  remainder = line[segment.next_index..] || ""
  line_indent = leading_space_count(line)
  recovered = segment.recovered

  if remainder.strip.empty? && !recovered
    scan_line_index = line_index + 1
    scan_line_number = line_number + 1
    scan_line_offset = line_offset + lines.fetch(line_index).bytesize

    while scan_line_index < lines.length
      raw_line = lines.fetch(scan_line_index)
      scan_line = raw_line.delete_suffix("\n").b
      segment_start = leading_space_count(scan_line)
      prefix = cstring ? 'c"' : '"'

      break unless segment_start > line_indent
      break if scan_line[segment_start].nil?
      break if scan_line[segment_start] == "#"
      break unless scan_line[segment_start, prefix.length] == prefix

      continued_segment = scan_string_segment(scan_line, segment_start, scan_line_number, cstring:, recover: @recovery_errors)
      continued_remainder = scan_line[continued_segment.next_index..] || ""
      break unless continued_remainder.strip.empty?

      value << continued_segment.value
      consumed_lines += 1
      last_line = scan_line
      last_line_number = scan_line_number
      last_line_offset = scan_line_offset
      last_segment_end = continued_segment.next_index
      last_line_has_newline = raw_line.end_with?("\n")
      recovered ||= continued_segment.recovered

      scan_line_offset += raw_line.bytesize
      scan_line_number += 1
      scan_line_index += 1

      break if continued_segment.recovered
    end
  end

  start_offset = line_offset + index
  if consumed_lines == 1
    lexeme = line[index...segment.next_index]
    @tokens << token(cstring ? :cstring : :string, lexeme, value, line_number, index + 1, start_offset:, end_offset: line_offset + segment.next_index)
    return StringLexResult.new(consumed_lines:, next_index: segment.next_index)
  end

  end_offset = last_line_offset + last_segment_end
  lexeme = @source.byteslice(start_offset, end_offset - start_offset)
  @tokens << token(cstring ? :cstring : :string, lexeme, value, line_number, index + 1, start_offset:, end_offset:)
  emit_line_newline(last_line, last_line_number, last_line_offset, last_line_has_newline)
  StringLexResult.new(consumed_lines:, next_index: last_segment_end)
end

#scan_string_segment(line, index, line_number, cstring: false, recover: false) ⇒ Object

Raises:



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/milk_tea/core/lexer/strings.rb', line 72

def scan_string_segment(line, index, line_number, cstring: false, recover: false)
  start = index
  index += cstring ? 2 : 1
  value = +""

  while index < line.length
    char = line[index]
    if char == '"'
      return StringSegment.new(next_index: index + 1, value:)
    end

    if char == "\\"
      escape = line[index + 1]
      if escape.nil?
        if recover
          @recovery_errors << LexError.new("unterminated string literal", line: line_number, column: start + 1, path: @path) if @recovery_errors
          return StringSegment.new(next_index: line.length, value:, recovered: true)
        end

        raise LexError.new("unterminated string literal", line: line_number, column: start + 1, path: @path)
      end

      value << decode_escape(escape)
      index += 2
      next
    end

    value << char
    index += 1
  end

  if recover
    @recovery_errors << LexError.new("unterminated string literal", line: line_number, column: start + 1, path: @path) if @recovery_errors
    return StringSegment.new(next_index: line.length, value:, recovered: true)
  end

  raise LexError.new("unterminated string literal", line: line_number, column: start + 1, path: @path)
end