Class: Secryst::Vocab

Inherits:
Object
  • Object
show all
Defined in:
lib/secryst/vocab.rb

Constant Summary collapse

UNK = 
"<unk>"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(list, specials: ["<unk>", "<pad>", "<sos>", "<eos>"], specials_first: true) ⇒ Vocab

Returns a new instance of Vocab.



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/secryst/vocab.rb', line 6

def initialize(
  list, specials: ["<unk>", "<pad>", "<sos>", "<eos>"], specials_first: true
)
  @unk_index = nil
  @itos = []
  if specials_first && (list & specials).length == 0
    @itos = specials
  end

  @itos += list

  if !specials_first && (list & specials).length == 0
    @itos.concat(specials)
  end

  # Automatic substitution of unknown symbols
  if @itos.include?("<unk>")
    unk_index = @itos.index("<unk>")
    @stoi = Hash.new(unk_index)
  elsif @itos.include?("[UNK]")
    unk_index = @itos.index("[UNK]")
    @stoi = Hash.new(unk_index)
  else
    @stoi = {}
  end


  # stoi is simply a reverse dict for itos
  @itos.each_with_index do |tok, i|
    @stoi[tok] = i
  end
end

Instance Attribute Details

#itos ⇒ Object (readonly)

Returns the value of attribute itos.



4
5
6
 
# File 'lib/secryst/vocab.rb', line 4

def itos
  @itos
end

#stoi ⇒ Object (readonly)

Returns the value of attribute stoi.



4
5
6
 
# File 'lib/secryst/vocab.rb', line 4

def stoi
  @stoi
end

Instance Method Details

#[](token) ⇒ Object 



39
40
41
 
# File 'lib/secryst/vocab.rb', line 39

def [](token)
  @stoi.fetch(token, @stoi.fetch(UNK))
end

#length ⇒ Object Also known as: size 



43
44
45
 
# File 'lib/secryst/vocab.rb', line 43

def length
  @itos.length
end