Class: TWPipeline::Stages::Tokenize

Inherits:
MapStage show all
Defined in:
lib/twpipeline/stages/tokenize.rb,
sig/twpipeline.rbs

Constant Summary collapse

FIT_LINES =

Returns:

  • (::Integer) 
200_000

Constants inherited from TWPipeline::Stage

TWPipeline::Stage::REGISTRY

Instance Attribute Summary

Attributes inherited from TWPipeline::Stage

#options, #policy, #resources

Instance Method Summary collapse

Methods inherited from MapStage

#process

Methods inherited from TWPipeline::Stage

#drop?, #flag, #initialize, lookup, order, ordered, register, #reject, slug

Constructor Details

This class inherits a constructor from TWPipeline::Stage

Instance Method Details

#call(input, output) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/twpipeline/stages/tokenize.rb', line 10

def call(input, output)
  head = []
  lines = input.each_line
  lines.each { |line|
    head << line
    break if head.length >= fit_lines
  }
  prime(head)

  Parallel.pipe(head.each + lines, output, workers: resources.cores) { |line| process(line) }
end

#transform(record) ⇒ Object 



22
 
# File 'lib/twpipeline/stages/tokenize.rb', line 22

def transform(record) = record.merge(tokens: segmenter.call(record.fetch(:text, "")))