30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
|
# File 'lib/yanagi/mora.rb', line 30
def self.tokenize(input)
hira = Normalize.to_hiragana(Normalize.nfkc(input.to_s))
moras = []
mora_map = Rules.mora_map
i = 0
len = hira.length
while i < len
c1 = hira[i]
c2 = hira[i, 2]
if c1 == "っ"
moras << Mora.new(kana: "っ", kind: :sokuon, onset: nil, nucleus: nil)
i += 1
elsif c1 == "ー"
moras << Mora.new(kana: "ー", kind: :chouonpu, onset: nil, nucleus: nil)
i += 1
elsif c1 == "ん"
moras << Mora.new(kana: "ん", kind: :moraic_n, onset: "n", nucleus: "n")
i += 1
elsif c2 && mora_map.key?(c2.to_sym)
info = mora_map[c2.to_sym]
moras << Mora.new(
kana: c2,
kind: :syllable,
onset: info[:onset]&.to_s || "",
nucleus: info[:nucleus]&.to_s || ""
)
i += 2
elsif mora_map.key?(c1.to_sym)
info = mora_map[c1.to_sym]
moras << Mora.new(
kana: c1,
kind: :syllable,
onset: info[:onset]&.to_s || "",
nucleus: info[:nucleus]&.to_s || ""
)
i += 1
else
moras << Mora.new(kana: c1, kind: :passthrough, onset: nil, nucleus: nil)
i += 1
end
end
moras
end
|