Class: LlmOptimizer::Compressor

Inherits:
Object
  • Object
show all
Defined in:
lib/llm_optimizer/compressor.rb

Constant Summary collapse

STOP_WORDS = 
%w[
  the a an is are was were be been being
  of in to for on at by with from as into
  through during before after above below
  between out off over under again further
  then once
].freeze
FENCE_RE = 
/(```[\s\S]*?```|~~~[\s\S]*?~~~)/

Instance Method Summary collapse

Constructor Details

#initialize(slm_client: nil) ⇒ Compressor

Returns a new instance of Compressor.



15
16
17
 
# File 'lib/llm_optimizer/compressor.rb', line 15

def initialize(slm_client: nil)
  @slm_client = slm_client
end

Instance Method Details

#compress(prompt) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/llm_optimizer/compressor.rb', line 19

def compress(prompt)
  segments = prompt.split(FENCE_RE)

  processed = segments.map.with_index do |segment, i|
    # Odd-indexed segments are fenced code blocks (captured group)
    if i.odd?
      segment
    else
      remove_stop_words(segment)
    end
  end

  result = processed.join
  result.gsub(/\s{2,}/, " ").strip
end

#estimate_tokens(text) ⇒ Object 



35
36
37
 
# File 'lib/llm_optimizer/compressor.rb', line 35

def estimate_tokens(text)
  (text.length / 4.0).ceil
end