8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
# File 'lib/milk_tea/core/lexer/heredocs.rb', line 8
def lex_heredoc(lines, line_index, index, line_number, line_offset, cstring:, format: false)
line = lines.fetch(line_index).delete_suffix("\n").b
prefix_length = heredoc_prefix(cstring:, format:).length
tag_start = index + prefix_length
tag_end = tag_start
tag_end += 1 while tag_end < line.length && identifier_part?(line[tag_end])
tag = line[tag_start...tag_end]
remainder = line[tag_end..] || ""
unless heredoc_context_allowed?
return lex_symbol(line, index, line_number, line_offset:)
end
unless remainder.strip.empty?
raise LexError.new("unexpected characters after heredoc tag", line: line_number, column: tag_end + 1, path: @path)
end
content_lines = []
terminator_line = nil
terminator_line_number = nil
terminator_line_offset = nil
terminator_has_newline = false
last_line = line
last_line_number = line_number
last_line_offset = line_offset
last_line_has_newline = lines.fetch(line_index).end_with?("\n")
resync_line_offset = nil
resync_line_number = nil
scan_line_number = line_number + 1
scan_line_offset = line_offset + lines.fetch(line_index).bytesize
scan_line_index = line_index + 1
content_min_indent = nil
while scan_line_index < lines.length
raw_line = lines.fetch(scan_line_index)
raw_text = raw_line.delete_suffix("\n")
if heredoc_terminator?(raw_text, tag)
terminator_line = raw_text
terminator_line_number = scan_line_number
terminator_line_offset = scan_line_offset
terminator_has_newline = raw_line.end_with?("\n")
break
end
if @recovery_errors && content_min_indent&.positive? && top_level_resync_line?(raw_text)
resync_line_offset = scan_line_offset
resync_line_number = scan_line_number
break
end
content_lines << raw_line
last_line = raw_text
last_line_number = scan_line_number
last_line_offset = scan_line_offset
last_line_has_newline = raw_line.end_with?("\n")
scan_line_offset += raw_line.bytesize
scan_line_number += 1
scan_line_index += 1
unless raw_text.strip.empty?
line_indent = leading_space_count(raw_text)
content_min_indent = line_indent if content_min_indent.nil? || line_indent < content_min_indent
end
end
if terminator_line.nil? && @recovery_errors && resync_line_number.nil? && scan_line_index < lines.length
trailing_text = lines.fetch(scan_line_index).delete_suffix("\n")
if top_level_resync_line?(trailing_text)
resync_line_offset = scan_line_offset
resync_line_number = scan_line_number
end
end
if terminator_line.nil?
if @recovery_errors
@recovery_errors << LexError.new("unterminated heredoc literal", line: line_number, column: index + 1, path: @path)
start_offset = line_offset + index
end_offset = resync_line_offset || scan_line_offset
lexeme = @source.byteslice(start_offset, end_offset - start_offset)
value = dedent_heredoc_content(content_lines)
content_margin = heredoc_content_margin(content_lines)
literal = if format
parse_format_heredoc_parts(value, start_line: line_number + 1, start_column: content_margin + 1)
else
value
end
token_type = if format
:fstring
elsif cstring
:cstring
else
:string
end
@tokens << token(token_type, lexeme, literal, line_number, index + 1, start_offset:, end_offset:)
emit_line_newline(last_line, last_line_number, last_line_offset, last_line_has_newline)
return resync_line_number ? (resync_line_number - line_number) : (scan_line_index - line_index)
end
raise LexError.new("unterminated heredoc literal", line: line_number, column: index + 1, path: @path)
end
start_offset = line_offset + index
end_offset = terminator_line_offset + terminator_line.bytesize
lexeme = @source.byteslice(start_offset, end_offset - start_offset)
value = dedent_heredoc_content(content_lines)
content_margin = heredoc_content_margin(content_lines)
literal = if format
parse_format_heredoc_parts(value, start_line: line_number + 1, start_column: content_margin + 1)
else
value
end
token_type = if format
:fstring
elsif cstring
:cstring
else
:string
end
@tokens << token(token_type, lexeme, literal, line_number, index + 1, start_offset:, end_offset:)
emit_line_newline(terminator_line, terminator_line_number, terminator_line_offset, terminator_has_newline)
(scan_line_index - line_index) + 1
end
|