Class: TWPipeline::Segmenter

Inherits:
Object
  • Object
show all
Defined in:
lib/twpipeline/segmenter.rb,
sig/twpipeline.rbs

Defined Under Namespace

Classes: Model

Constant Summary collapse

MAX_WORD =

Returns:

  • (::Integer) 
8
HAN_RUN =

Returns:

  • (::Regexp) 
/[\u{3400}-\u{9FFF}]+/

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(words:, chars: Set.new, model: nil) ⇒ Segmenter

Returns a new instance of Segmenter.

Parameters:

  • words: (::Set[::String])
  • chars: (::Set[::String]) (defaults to: Set.new)
  • model: (Model, nil) (defaults to: nil) 


25
26
27
28
29
30
 
# File 'lib/twpipeline/segmenter.rb', line 25

def initialize(words:, chars: Set.new, model: nil)
  @words = words
  @chars = chars
  @limit = [words.map(&:length).max.to_i, MAX_WORD].min
  @model = model
end

Instance Attribute Details

#model ⇒ Model? (readonly)

Returns the value of attribute model.

Returns:



23
24
25
 
# File 'lib/twpipeline/segmenter.rb', line 23

def model
  @model
end

#words ⇒ ::Set[::String] (readonly)

Returns the value of attribute words.

Returns:

  • (::Set[::String]) 


23
24
25
 
# File 'lib/twpipeline/segmenter.rb', line 23

def words
  @words
end

Class Method Details

.entries(value) ⇒ ::Set[::String]

Parameters:

  • value (Object)

Returns:

  • (::Set[::String]) 


18
 
# File 'lib/twpipeline/segmenter.rb', line 18

def entries(value) = (value.is_a?(Hash) ? value.keys : Array(value)).to_set

.load(path) ⇒ Segmenter

Parameters:

  • path (Object)

Returns:



13
14
15
16
 
# File 'lib/twpipeline/segmenter.rb', line 13

def load(path)
  data = JSON.parse(Pathname(path).read)
  new(words: entries(data.fetch("words")), chars: entries(data.fetch("chars", [])))
end

.runs(text) ⇒ ::Array[::String]

Parameters:

  • text (::String)

Returns:

  • (::Array[::String]) 


20
 
# File 'lib/twpipeline/segmenter.rb', line 20

def runs(text) = text.scan(HAN_RUN)

Instance Method Details

#call(text) ⇒ ::Array[::String]

Parameters:

  • text (::String)

Returns:

  • (::Array[::String]) 


43
 
# File 'lib/twpipeline/segmenter.rb', line 43

def call(text) = self.class.runs(text).flat_map { |run| segment(run) }

#fit(runs, rounds: 2) ⇒ self

Parameters:

  • runs (::Array[::String])
  • rounds: (::Integer) (defaults to: 2)

Returns:

  • (self) 


32
33
34
35
36
37
38
39
40
41
 
# File 'lib/twpipeline/segmenter.rb', line 32

def fit(runs, rounds: 2)
  counts = naive_counts(runs)
  rounds.times do
    @model = build_model(*counts)
    counts = recount(runs)
  end

  @model = build_model(*counts)
  self
end

#segment(run) ⇒ ::Array[::String]

Parameters:

  • run (::String)

Returns:

  • (::Array[::String]) 


45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/twpipeline/segmenter.rb', line 45

def segment(run)
  chars = run.chars
  size = chars.length
  best = Array.new(size + 1, Float::INFINITY)
  best[0] = 0.0
  back = Array.new(size + 1, 0)

  (1..size).each do |stop|
    ([1, stop - @model.max_word + 1].max..stop).each do |start|
      previous = best[start - 1]
      next if previous.infinite?

      token = chars[(start - 1)...stop].join
      price = price_of(token)
      next if price.nil?

      value = previous + price
      next unless value < best[stop]

      best[stop] = value
      back[stop] = start - 1
    end
  end

  unwind(chars, back, size)
end