Class: Rogalik::Pipeline

Inherits:
Object
  • Object
show all
Defined in:
lib/rogalik.rb

Overview

Orchestrates the embedding and storage pipeline.

Plugins are registered in PLUGINS and must implement the interface defined by their corresponding stage class.

Defined Under Namespace

Classes: Chunker, Embeddings, Loader, PiiDetector, VectorStore

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, logger: nil) ⇒ Pipeline

Parameters:

  • source (String, Pathname)
  • logger (Logger) (defaults to: nil) 


138
139
140
141
142
143
144
145
146
 
# File 'lib/rogalik.rb', line 138

def initialize(source, logger: nil)
  @source = source
  @logger = logger || Rogalik.configuration.logger
  @loader = Loader.new(source, logger)
  @chunker = Chunker.new
  @embeddings = Embeddings.new
  @vector_store = VectorStore.new
  @pii_detector = PiiDetector.new
end

Instance Attribute Details

#chunker ⇒ Object (readonly)

Returns the value of attribute chunker.



148
149
150
 
# File 'lib/rogalik.rb', line 148

def chunker
  @chunker
end

#embeddings ⇒ Object (readonly)

Returns the value of attribute embeddings.



148
149
150
 
# File 'lib/rogalik.rb', line 148

def embeddings
  @embeddings
end

#loader ⇒ Object (readonly)

Returns the value of attribute loader.



148
149
150
 
# File 'lib/rogalik.rb', line 148

def loader
  @loader
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



148
149
150
 
# File 'lib/rogalik.rb', line 148

def logger
  @logger
end

#pii_detector ⇒ Object (readonly)

Returns the value of attribute pii_detector.



148
149
150
 
# File 'lib/rogalik.rb', line 148

def pii_detector
  @pii_detector
end

#source ⇒ Object (readonly)

Returns the value of attribute source.



148
149
150
 
# File 'lib/rogalik.rb', line 148

def source
  @source
end

#vector_store ⇒ Object (readonly)

Returns the value of attribute vector_store.



148
149
150
 
# File 'lib/rogalik.rb', line 148

def vector_store
  @vector_store
end

Class Method Details

.plugin(mod) ⇒ void

This method returns an undefined value.

Parameters:

  • mod (Module, Symbol) —

    ]

  • args (Array) 


80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/rogalik.rb', line 80

def self.plugin(mod, ...)
  if mod.is_a?(Symbol)
    require "rogalik/plugins/#{mod}"
    mod = PLUGINS.fetch(mod)
  end

  mod.before_load(self, ...) if mod.respond_to?(:before_load)

  if defined?(mod::LoaderMethods)
    self::Loader.include(mod::LoaderMethods)
  end

  if defined?(mod::EmbeddingsMethods)
    self::Embeddings.include(mod::EmbeddingsMethods)
  end

  if defined?(mod::VectorStoreMethods)
    self::VectorStore.include(mod::VectorStoreMethods)
  end

  if defined?(mod::ChunkerMethods)
    self::Chunker.include(mod::ChunkerMethods)
  end

  if defined?(mod::PiiDetectorMethods)
    self::PiiDetector.include(mod::PiiDetectorMethods)
  end

  if defined?(mod::LoaderClassMethods)
    self::Loader.extend(mod::LoaderClassMethods)
  end

  if defined?(mod::EmbeddingsClassMethods)
    self::Embeddings.extend(mod::EmbeddingsClassMethods)
  end

  if defined?(mod::VectorStoreClassMethods)
    self::VectorStore.extend(mod::VectorStoreClassMethods)
  end

  if defined?(mod::ChunkerClassMethods)
    self::Chunker.extend(mod::ChunkerClassMethods)
  end

  if defined?(mod::PiiDetectorClassMethods)
    self::PiiDetector.extend(mod::PiiDetectorClassMethods)
  end

  mod.after_load(self, ...) if mod.respond_to?(:after_load)
end

.register_plugin(name, mod) ⇒ Object

Parameters:

  • name (Symbol) 


71
72
73
 
# File 'lib/rogalik.rb', line 71

def self.register_plugin(name, mod)
  PLUGINS[name] = mod
end

Instance Method Details

#run ⇒ void

This method returns an undefined value.

Executes the pipeline stages in order.



159
160
161
162
163
164
165
 
# File 'lib/rogalik.rb', line 159

def run
  loader.load
  chunker.chunk
  embeddings.embeddings
  vector_store.add_documents
  pii_detector.anonymize
end