Class: ClaudeMemory::Index::LexicalFTS

Inherits:
Object
  • Object
show all
Defined in:
lib/claude_memory/index/lexical_fts.rb

Defined Under Namespace

Classes: CorruptRankIndexError

Constant Summary collapse

RANK_CORRUPTION_HINT = 
"FTS5 rank index is corrupt — recall is broken even though " \
"the database otherwise looks healthy. Run `claude-memory compact` to rebuild it."

Instance Method Summary collapse

Constructor Details

#initialize(store) ⇒ LexicalFTS

Returns a new instance of LexicalFTS.



15
16
17
18
19
20
 
# File 'lib/claude_memory/index/lexical_fts.rb', line 15

def initialize(store)
  @store = store
  @db = store.db
  @fts_table_ensured = false
  @contentless = nil
end

Instance Method Details

#escape_fts_query(query) ⇒ Object 



120
121
122
123
124
125
126
127
128
129
 
# File 'lib/claude_memory/index/lexical_fts.rb', line 120

def escape_fts_query(query)
  words = query.split(/\s+/).map do |word|
    next word if word == "*"
    escaped = word.gsub('"', '""')
    %("#{escaped}")
  end.compact

  return words.first if words.size == 1
  words.join(" OR ")
end

#index_content_item(content_item_id, text) ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/claude_memory/index/lexical_fts.rb', line 22

def index_content_item(content_item_id, text)
  ensure_fts_table!
  if contentless?
    existing = @db.fetch("SELECT rowid FROM content_fts WHERE rowid = ?", content_item_id).first
    return if existing
    @db.fetch("INSERT INTO content_fts(rowid, text) VALUES (?, ?)", content_item_id, text).insert
  else
    existing = @db[:content_fts].where(content_item_id: content_item_id).get(:content_item_id)
    return if existing
    @db[:content_fts].insert(content_item_id: content_item_id, text: text)
  end
end

#rebuild! ⇒ Object

Rebuild the entire FTS index from content_items. Always rebuilds as contentless to save space.



105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/claude_memory/index/lexical_fts.rb', line 105

def rebuild!
  @db.run("DROP TABLE IF EXISTS content_fts")
  @fts_table_ensured = false
  @contentless = nil

  create_contentless_table!

  @db[:content_items].select(:id, :raw_text).order(:id).paged_each(rows_per_fetch: 500) do |row|
    @db.fetch(
      "INSERT INTO content_fts(rowid, text) VALUES (?, ?)",
      row[:id], row[:raw_text]
    ).insert
  end
end

#remove_content_item(content_item_id, text) ⇒ Object

Remove a content item from the FTS index



91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/claude_memory/index/lexical_fts.rb', line 91

def remove_content_item(content_item_id, text)
  ensure_fts_table!
  if contentless?
    @db.fetch(
      "INSERT INTO content_fts(content_fts, rowid, text) VALUES('delete', ?, ?)",
      content_item_id, text
    ).insert
  else
    @db[:content_fts].where(content_item_id: content_item_id).delete
  end
end

#search(query, limit: 20) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/claude_memory/index/lexical_fts.rb', line 35

def search(query, limit: 20)
  ensure_fts_table!
  return [] if query.nil? || query.strip.empty?

  if query.strip == "*"
    return @db[:content_items]
        .order(Sequel.desc(:id))
        .limit(limit)
        .select_map(:id)
  end

  escaped_query = escape_fts_query(query)
  with_rank_index do
    if contentless?
      @db.fetch(
        "SELECT rowid AS content_item_id FROM content_fts WHERE text MATCH ? ORDER BY rank LIMIT ?",
        escaped_query, limit
      ).map { |row| row[:content_item_id] }
    else
      @db[:content_fts]
        .where(Sequel.lit("text MATCH ?", escaped_query))
        .order(:rank)
        .limit(limit)
        .select_map(:content_item_id)
    end
  end
end

#search_with_ranks(query, limit: 20) ⇒ Array<Hash>

Search returning content IDs with FTS5 BM25 rank values

Parameters:

  • query (String) —

    Search query

  • limit (Integer) (defaults to: 20) —

    Maximum results

Returns:

  • (Array<Hash>) —

    Results with :content_item_id and :rank



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/claude_memory/index/lexical_fts.rb', line 67

def search_with_ranks(query, limit: 20)
  ensure_fts_table!
  return [] if query.nil? || query.strip.empty?
  return [] if query.strip == "*"

  escaped_query = escape_fts_query(query)
  with_rank_index do
    if contentless?
      @db.fetch(
        "SELECT rowid AS content_item_id, rank FROM content_fts WHERE text MATCH ? ORDER BY rank LIMIT ?",
        escaped_query, limit
      ).all
    else
      @db[:content_fts]
        .where(Sequel.lit("text MATCH ?", escaped_query))
        .order(:rank)
        .limit(limit)
        .select(Sequel.lit("content_item_id, rank"))
        .all
    end
  end
end