Class: Terret::LLM::Service

Inherits:
Hames::Service
  • Object
show all
Defined in:
lib/terret/llm.rb

Overview

ctx.llm — the adapter seam. llm/stream is a waterfall wrapping every request: middleware may rewrite the request or replace the stream.

Instance Method Summary collapse

Instance Method Details

#register_adapter(name, adapter) ⇒ Object 



31
32
33
34
 
# File 'lib/terret/llm.rb', line 31

def register_adapter(name, adapter)
  @adapters[name.to_s] = adapter
  -> { @adapters.delete(name.to_s) }
end

#resolve(role) ⇒ Object 



36
37
38
39
40
 
# File 'lib/terret/llm.rb', line 36

def resolve(role)
  spec = @roles.fetch(role) { raise KeyError, "no model role #{role.inspect}" }
  provider, model = spec.split("/", 2)
  [@adapters.fetch(provider), model]
end

#start(_ctx) ⇒ Object 



26
27
28
29
 
# File 'lib/terret/llm.rb', line 26

def start(_ctx)
  @adapters = {}
  @roles    = config[:roles] || {} # :main => "fake/scripted"
end

#stream(ctx, role:, request:, &on_event) ⇒ Object

Streams the request through the llm/stream waterfall; the base of the waterfall invokes the resolved adapter. Yields StreamEvents, returns the final assistant Message.



45
46
47
48
49
50
51
 
# File 'lib/terret/llm.rb', line 45

def stream(ctx, role:, request:, &on_event)
  adapter, model = resolve(role)
  req = request.with(model: model)
  ctx.waterfall("llm/stream", req) do |final_req|
    adapter.stream(final_req, &on_event)
  end
end