Class: TWPipeline::Stages::Normalize

Inherits:
MapStage show all
Defined in:
lib/twpipeline/stages/normalize.rb,
sig/twpipeline.rbs

Constant Summary

Constants inherited from TWPipeline::Stage

TWPipeline::Stage::REGISTRY

Instance Attribute Summary

Attributes inherited from TWPipeline::Stage

#options, #policy, #resources

Instance Method Summary collapse

Methods inherited from MapStage

#call, #process

Methods inherited from TWPipeline::Stage

#call, #drop?, #flag, #initialize, lookup, order, ordered, register, #reject, slug

Constructor Details

This class inherits a constructor from TWPipeline::Stage

Instance Method Details

#transform(record) ⇒ Object 



8
9
10
11
12
 
# File 'lib/twpipeline/stages/normalize.rb', line 8

def transform(record)
  original = record.fetch(:text, "")
  normalized = TWFilter.normalize(original, collapse_repeats: options.fetch(:collapse_repeats, true))
  record.merge(text: normalized, normalized: normalized != original)
end