Module: Legion::Extensions::Knowledge::Runners::Corpus

Included in:
Client
Defined in:
lib/legion/extensions/knowledge/runners/corpus.rb

Overview

rubocop:disable Legion/Extension/RunnerIncludeHelpers

Class Method Summary collapse

Class Method Details

.corpus_stats(path:, extensions: nil) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/legion/extensions/knowledge/runners/corpus.rb', line 14

def corpus_stats(path:, extensions: nil)
  return { success: false, error: 'path does not exist' } unless ::File.exist?(path)

  opts = { path: path }
  opts[:extensions] = extensions if extensions
  entries = Helpers::Manifest.scan(**opts)
  chunk_count = entries.sum do |entry|
    sections = Helpers::Parser.parse(file_path: entry[:path])
    next 0 if sections.first&.key?(:error)

    Helpers::Chunker.chunk(sections: sections).size
  end

  {
    success:          true,
    path:             path,
    file_count:       entries.size,
    estimated_chunks: chunk_count,
    total_bytes:      entries.sum { |e| e[:size] }
  }
rescue StandardError => e
  { success: false, error: e.message }
end

.manifest_path(path:) ⇒ Object 



10
11
12
 
# File 'lib/legion/extensions/knowledge/runners/corpus.rb', line 10

def manifest_path(path:)
  Helpers::ManifestStore.store_path(corpus_path: path)
end