Class: CharSplit::NgramCounter

Inherits:
Object
  • Object
show all
Defined in:
lib/charsplit/ngram_counter.rb

Overview

Counts where character n-grams occur in a collection of words.

Constant Summary collapse

POSITIONS = 
%i[prefix infix suffix].freeze

Instance Method Summary collapse

Constructor Details

#initialize(min_length: 3, max_length: 20) ⇒ NgramCounter

Returns a new instance of NgramCounter.



8
9
10
11
12
13
 
# File 'lib/charsplit/ngram_counter.rb', line 8

def initialize(min_length: 3, max_length: 20)
  @min_length = min_length
  @max_length = max_length
  @positional_counts = POSITIONS.to_h { |position| [position, Hash.new(0)] }
  @total_counts = Hash.new(0)
end

Instance Method Details

#add(word) ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/charsplit/ngram_counter.rb', line 15

def add(word)
  characters = word.downcase.chars
  middle = characters[1...-1] || []

  (@min_length..@max_length).each do |length|
    # This deliberately retains CharSplit's behavior for words shorter than
    # +length+: Python's slices return the complete word in that case.
    count(:prefix, characters.first(length).join)
    count(:suffix, characters.last(length).join)

    middle.each_cons(length) { |ngram| count(:infix, ngram.join) }
  end

  self
end

#probabilities ⇒ Object 



31
32
33
34
35
36
37
38
 
# File 'lib/charsplit/ngram_counter.rb', line 31

def probabilities
  POSITIONS.to_h do |position|
    probabilities = @positional_counts.fetch(position).each_with_object({}) do |(ngram, count), result|
      result[ngram] = count.fdiv(@total_counts.fetch(ngram)) if count > 1
    end
    [position.to_s, probabilities]
  end
end