Class: Legion::Extensions::Llm::StreamAccumulator

Inherits:
Object
  • Object
show all
Includes:
Logging::Helper
Defined in:
lib/legion/extensions/llm/stream_accumulator.rb

Overview

Assembles streaming responses from LLMs into complete messages.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ StreamAccumulator

Returns a new instance of StreamAccumulator.



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 12

def initialize
  @content = +''
  @thinking_text = +''
  @thinking_signature = nil
  @tool_calls = {}
  @stop_reason = nil
  @input_tokens = nil
  @output_tokens = nil
  @cached_tokens = nil
  @cache_creation_tokens = nil
  @thinking_tokens = nil
  @inside_think_tag = false
  @pending_think_tag = +''
  @active_think_close_tag = nil
  @untagged_preamble_pending = true
  @untagged_preamble_buffer = +''
  @latest_tool_call_id = nil
end

Instance Attribute Details

#content ⇒ Object (readonly)

Returns the value of attribute content.



10
11
12
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 10

def content
  @content
end

#model_id ⇒ Object (readonly)

Returns the value of attribute model_id.



10
11
12
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 10

def model_id
  @model_id
end

#stop_reason ⇒ Object (readonly)

Returns the value of attribute stop_reason.



10
11
12
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 10

def stop_reason
  @stop_reason
end

#tool_calls ⇒ Object (readonly)

Returns the value of attribute tool_calls.



10
11
12
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 10

def tool_calls
  @tool_calls
end

Instance Method Details

#add(chunk) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 31

def add(chunk)
  log.debug { chunk.inspect } if Legion::Extensions::Llm.config.log_stream_debug
  @model_id ||= chunk.model_id

  @last_content_delta = +''
  @last_thinking_delta = +''
  handle_chunk_content(chunk)
  append_thinking_from_chunk(chunk)
  count_tokens chunk
  @stop_reason = chunk.stop_reason if chunk.respond_to?(:stop_reason) && chunk.stop_reason
  log.debug { inspect } if Legion::Extensions::Llm.config.log_stream_debug
end

#filtered_chunk(chunk) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 44

def filtered_chunk(chunk)
  has_content = !@last_content_delta.empty?
  has_thinking = !@last_thinking_delta.empty?
  has_tokens = chunk.input_tokens&.positive? || chunk.output_tokens&.positive?
  return nil unless has_content || has_thinking || chunk.tool_call? || has_tokens

  Chunk.new(
    role: :assistant,
    content: has_content ? @last_content_delta : nil,
    thinking: has_thinking ? Thinking.build(text: @last_thinking_delta) : chunk.thinking,
    model_id: chunk.model_id,
    tool_calls: chunk.tool_calls,
    input_tokens: chunk.input_tokens,
    output_tokens: chunk.output_tokens,
    raw: chunk.raw
  )
end

#flush_pending_chunk ⇒ Object

Flush any text still held in the untagged-preamble buffer as a final streamed chunk. Without this, short responses that match the untagged-reasoning heuristic (e.g. starting with "I", "The", "Let me") and never hit a double newline are buffered for the entire stream and the caller's block never receives a single delta.



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 67

def flush_pending_chunk
  return nil if @untagged_preamble_buffer.empty?

  @last_content_delta = +''
  @last_thinking_delta = +''
  flush_pending_untagged_preamble_into_deltas
  return nil if @last_content_delta.empty? && @last_thinking_delta.empty?

  Chunk.new(
    role: :assistant,
    content: @last_content_delta.empty? ? nil : @last_content_delta,
    thinking: @last_thinking_delta.empty? ? nil : Thinking.build(text: @last_thinking_delta),
    model_id: model_id
  )
end

#to_message(response) ⇒ Object 



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/legion/extensions/llm/stream_accumulator.rb', line 83

def to_message(response)
  flush_pending_untagged_preamble

  if content.length < 50
    log.debug '[llm][stream_accumulator] action=short_content_debug ' \
              "content=#{content.inspect} thinking_chars=#{@thinking_text.length} " \
              "tool_calls=#{tool_calls.size} " \
              "inside_think_tag=#{@inside_think_tag} " \
              "pending_think_tag=#{@pending_think_tag.inspect} " \
              "untagged_preamble_pending=#{@untagged_preamble_pending} " \
              "thinking_start=#{@thinking_text[0, 200].inspect} " \
              "thinking_end=#{@thinking_text[-200..].inspect}"
    if content.length < 5 && @thinking_text.length > 100 && tool_calls.empty?
      log.debug '[llm][stream_accumulator] action=POSSIBLE_CONTENT_EATEN ' \
                "full_thinking=#{@thinking_text.inspect}"
    end
  end

  Message.new(
    role: :assistant,
    content: content.empty? ? nil : content,
    thinking: Thinking.build(
      text: @thinking_text.empty? ? nil : @thinking_text,
      signature: @thinking_signature
    ),
    tokens: Tokens.build(
      input: @input_tokens,
      output: @output_tokens,
      cached: @cached_tokens,
      cache_creation: @cache_creation_tokens,
      thinking: @thinking_tokens
    ),
    model_id: model_id,
    tool_calls: tool_calls_from_stream,
    stop_reason: @stop_reason,
    raw: response
  )
end