Class: ClaudeMemory::Core::TokenEstimator

Inherits:
Object
  • Object
show all
Defined in:
lib/claude_memory/core/token_estimator.rb

Constant Summary collapse

CHARS_PER_TOKEN =

Approximation: ~4 characters per token for English text More accurate for Claude’s tokenizer than simple word count

4.0

Class Method Summary collapse

Class Method Details

.estimate(text) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
 
# File 'lib/claude_memory/core/token_estimator.rb', line 10

def self.estimate(text)
  return 0 if text.nil? || text.empty?

  # Remove extra whitespace and count characters
  normalized = text.strip.gsub(/\s+/, " ")
  chars = normalized.length

  # Return ceiling to avoid underestimation
  (chars / CHARS_PER_TOKEN).ceil
end

.estimate_fact(fact) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
 
# File 'lib/claude_memory/core/token_estimator.rb', line 21

def self.estimate_fact(fact)
  # Estimate tokens for a fact record
  text = [
    fact[:subject_name],
    fact[:predicate],
    fact[:object_literal]
  ].compact.join(" ")

  estimate(text)
end