Module: TWPipeline::Frequency

Defined in:
lib/twpipeline/frequency.rb,
sig/twpipeline.rbs

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.dispersion(counts, sizes) ⇒ Object 



7
8
9
10
11
12
13
 
# File 'lib/twpipeline/frequency.rb', line 7

def dispersion(counts, sizes)
  total = counts.values.sum.to_f
  corpus = sizes.values.sum.to_f
  return 0.0 if total.zero? || corpus.zero?

  0.5 * sizes.sum { |part, size| (counts.fetch(part, 0) / total - size / corpus).abs }
end

.each_key(path) {|[current, parts]| ... } ⇒ Object

Yields:

  • ([current, parts]) 


15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/twpipeline/frequency.rb', line 15

def each_key(path)
  return to_enum(:each_key, path) unless block_given?

  current = nil
  parts = Hash.new(0)

  Pathname(path).each_line do |line|
    count, key, part = line.chomp.split("\t")
    if key != current && !current.nil?
      yield [current, parts]
      parts = Hash.new(0)
    end

    current = key
    parts[part] += count.to_i
  end

  yield [current, parts] unless current.nil?
end

.table(path, sizes) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/twpipeline/frequency.rb', line 35

def table(path, sizes)
  each_key(path).map do |key, parts|
    total = parts.values.sum
    {
      key: key,
      frequency: total,
      dispersion: dispersion(parts, sizes),
      adjusted: (total * (1 - dispersion(parts, sizes))).round(2),
      parts: parts.length
    }
  end
end

Instance Method Details

#self?.dispersion ⇒ ::Float

Parameters:

  • counts (::Hash[::String?, ::Integer])
  • sizes (::Hash[::String?, ::Integer])

Returns:

  • (::Float) 


95
 
# File 'sig/twpipeline.rbs', line 95

def self?.dispersion: (::Hash[::String?, ::Integer] counts, ::Hash[::String?, ::Integer] sizes) -> ::Float

#self? ⇒ void #self? ⇒ ::Enumerator[[::String, ::Hash[::String?, ::Integer]], void]

Overloads:

  • #self? ⇒ void

    This method returns an undefined value.

    Parameters:

    • path (Object)
  • #self? ⇒ ::Enumerator[[::String, ::Hash[::String?, ::Integer]], void]

    Parameters:

    • path (Object)

    Returns:

    • (::Enumerator[[::String, ::Hash[::String?, ::Integer]], void])

Yields:

Yield Parameters:

  • arg0 ([::String, ::Hash[::String?, ::Integer]])

Yield Returns:

  • (void) 


96
97
 
# File 'sig/twpipeline.rbs', line 96

def self?.each_key: (untyped path) { ([::String, ::Hash[::String?, ::Integer]]) -> void } -> void
| (untyped path) -> ::Enumerator[[::String, ::Hash[::String?, ::Integer]], void]

#self?.table ⇒ ::Array[::Hash[::Symbol, untyped]]

Parameters:

  • path (Object)
  • sizes (::Hash[::String?, ::Integer])

Returns:

  • (::Array[::Hash[::Symbol, untyped]]) 


98
 
# File 'sig/twpipeline.rbs', line 98

def self?.table: (untyped path, ::Hash[::String?, ::Integer] sizes) -> ::Array[::Hash[::Symbol, untyped]]