Module: Cohere::Transcribe::Alignment::Text

Defined in:
lib/cohere/transcribe/alignment/text.rb

Overview

Arabic/English normalization used by the pinned MMS aligner. The punctuation inventory mirrors the retained Python reference, including its corpus-derived private-use and invalid-codepoint entries.

Constant Summary collapse

MMS_PUNCTUATION_HEX = 
mms_punctuation_hex.tr("\n", " ").freeze
PREFIX_CODEPOINTS = 
[
  *".?,:!{}“”«»‹›„‚‟‛’‘><¿。;՚՛՜՝՞՟։¡،、।\"؛؟，！？；：（）﹏⋯、‧／～─＿".codepoints,
  *(0x300C..0x300F), *(0xFF41..0xFF44), *(0x3008..0x300B)
].freeze
PUNCTUATION_CODEPOINTS = 
begin
  values = MMS_PUNCTUATION_HEX.split.flat_map do |token|
    first, last = token.split("-", 2).map { |hex| Integer(hex, 16) }
    last ? (first..last).to_a : [first]
  end
  (values + PREFIX_CODEPOINTS).to_h { |codepoint| [codepoint, true] }.freeze
end
DELETION_CODEPOINTS = 
[
  0x200E, 0x200C, *(0x0656..0x0657), 0x200B, *(0x064B..0x0652),
  0x202C, 0x200F, 0x202A
].to_h { |codepoint| [codepoint, true] }.freeze
ARABIC_ROMANIZATION = 
{
  "ء" => "'", "آ" => "a", "أ" => "a", "ؤ" => "w", "إ" => "i", "ئ" => "ye",
  "ا" => "a", "ب" => "b", "ة" => "a", "ت" => "t", "ث" => "th", "ج" => "j",
  "ح" => "h", "خ" => "kh", "د" => "d", "ذ" => "th", "ر" => "r", "ز" => "z",
  "س" => "s", "ش" => "sh", "ص" => "s", "ض" => "d", "ط" => "t", "ظ" => "z",
  "ع" => "'", "غ" => "gh", "ف" => "f", "ق" => "q", "ك" => "k", "ل" => "l",
  "م" => "m", "ن" => "n", "ه" => "h", "و" => "w", "ى" => "a", "ي" => "y",
  "ٮ" => "b", "ٯ" => "q", "ٹ" => "tt", "ٺ" => "tt", "ٻ" => "b", "ټ" => "t",
  "ٽ" => "t", "پ" => "p", "ٿ" => "t", "ڀ" => "b", "ځ" => "z", "ڂ" => "h",
  "ڃ" => "ny", "ڄ" => "dy", "څ" => "ts", "چ" => "tch", "ڇ" => "tch",
  "ڈ" => "dd", "ډ" => "d", "ڊ" => "d", "ڋ" => "d", "ڌ" => "d", "ڍ" => "dd",
  "ڎ" => "d", "ڏ" => "d", "ڐ" => "d", "ڑ" => "rr", "ژ" => "j", "ښ" => "kh",
  "ڤ" => "v", "ک" => "k", "ڪ" => "k", "ګ" => "g", "ڭ" => "ng", "گ" => "g",
  "ں" => "n", "ھ" => "h", "ہ" => "h", "ۃ" => "a", "ۇ" => "u", "ۈ" => "yu",
  "ۋ" => "v", "ی" => "i", "ۍ" => "y", "ێ" => "y", "ې" => "e", "ے" => "y",
  "ە" => "ae", "ٴ" => "h"
}.freeze
LATIN_ROMANIZATION = 
{
  "æ" => "ae", "œ" => "oe", "ø" => "oe", "ß" => "ss", "ð" => "d", "þ" => "th",
  "ł" => "l", "đ" => "d", "ħ" => "h", "ı" => "i", "ə" => "e"
}.freeze
UROMAN_SEQUENCE_ROMANIZATION =

Multi-codepoint rules from Uroman 1.3.1.1 that are reachable from the pinned ASR tokenizers. Uroman chooses these before its single-character candidates; keeping them here preserves that longest-match behavior.

{
  "λμπ" => "lb", "νμπ" => "nb", "ρμπ" => "rb",
  "γγ" => "ng", "ει" => "ei", "ευ" => "eu", "αυ" => "au", "ου" => "ou",
  "ηυ" => "eu", "υι" => "ui", "ωυ" => "ou", "ντ" => "nd",
  "シャ" => "sha", "シュ" => "shu", "ショ" => "sho",
  "チャ" => "cha", "チェ" => "che", "チュ" => "chu", "チョ" => "cho",
  "ジャ" => "ja", "ジュ" => "ju", "ジョ" => "jo", "ジェ" => "je",
  "ヂャ" => "ja", "ヂュ" => "ju", "ヂョ" => "jo",
  "フェ" => "fe", "ヴェ" => "ve", "フィ" => "fi", "ウィ" => "wi",
  "ヴィ" => "vi", "ティ" => "ti", "ディ" => "di",
  "しゃ" => "sha", "しゅ" => "shu", "しょ" => "sho",
  "ちゃ" => "cha", "ちゅ" => "chu", "ちょ" => "cho",
  "じゃ" => "ja", "じゅ" => "ju", "じょ" => "jo",
  "ぢゃ" => "ja", "ぢゅ" => "ju", "ぢょ" => "jo"
}.freeze
UROMAN_SEQUENCE_LENGTHS = 
UROMAN_SEQUENCE_ROMANIZATION.keys.map(&:length).uniq.sort.reverse.freeze
JAPANESE_SMALL_Y = 
"ゃゅょャュョ".chars.freeze
JAPANESE_SMALL_TSU = 
"っッ".chars.freeze
JAPANESE_SCRIPT = 
{
  "ゃ" => :hiragana, "ゅ" => :hiragana, "ょ" => :hiragana,
  "ャ" => :katakana, "ュ" => :katakana, "ョ" => :katakana
}.freeze
DIGIT_PATTERN = 
"0-9০-৯០-៩०-९୦-୯۰-۹꤀-꤉０-９൦-൯၀-၉ⅰ-ⅹ"

Class Method Summary collapse

Class Method Details

.normalize(text) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/cohere/transcribe/alignment/text.rb', line 102

def normalize(text)
  normalized = PythonText.nfkc_lower(text.to_s)
  normalized = normalized.gsub(/\([^)]*\p{Nd}[^)]*\)/, " ")
  normalized = normalized.gsub("&lt;", "").gsub("&gt;", "").gsub("&nbsp", "")
  normalized = normalized.gsub(/(\S+)[\u201B\u2019\u2018](\S+)/, "\\1'\\2")
  normalized = normalized.tr("ٱٰۥۦ", "ااوي")
  normalized = normalized.delete("ـٓ")
  normalized = normalized.tr("ٕٔ", "ءء")
  normalized = normalized.each_char.map do |character|
    PUNCTUATION_CODEPOINTS.key?(character.ord) ? " " : character
  end.join
  normalized = normalized.each_char.reject do |character|
    DELETION_CODEPOINTS.key?(character.ord)
  end.join
  whitespace = PythonText.whitespace_class
  normalized = normalized.gsub(/\A[#{DIGIT_PATTERN}]+(?=#{whitespace})/, " ")
  normalized = normalized.gsub(/(?<=#{whitespace})[#{DIGIT_PATTERN}]+(?=#{whitespace}|\z)/, " ")
  PythonText.collapse(normalized)
end

.postprocess(text_starred, spans, stride_ms) ⇒ Object 



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/cohere/transcribe/alignment/text.rb', line 141

def postprocess(text_starred, spans, stride_ms)
  results = []
  text_starred.each_with_index do |text, index|
    next if text == "<star>"

    span = spans.fetch(index)
    results << {
      start: span.first.start * stride_ms / 1_000.0,
      end: (span.last.end + 1) * stride_ms / 1_000.0,
      text: text
    }
  end
  results.each_cons(2) do |current, following|
    following[:start] = current[:end] if following[:start] < current[:end]
  end
  results.freeze
end

.preprocess(text, language) ⇒ Object 



129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/cohere/transcribe/alignment/text.rb', line 129

def preprocess(text, language)
  words = PythonText.split(text.to_s)
  tokens = words.map { |word| romanize(word, language) }
  token_stream = []
  text_stream = []
  tokens.zip(words).each do |token, word|
    token_stream.push("<star>", token)
    text_stream.push("<star>", word)
  end
  [token_stream.freeze, text_stream.freeze]
end

.romanize(text, language) ⇒ Object 



122
123
124
125
126
127
 
# File 'lib/cohere/transcribe/alignment/text.rb', line 122

def romanize(text, language)
  normalized = normalize(text)
  transliterated = romanize_normalized(normalized, language)
  spaced = PythonText.strip(transliterated).each_char.to_a.join(" ").downcase
  PythonText.strip(spaced.gsub(/[^a-z' ]/, " ").gsub(/ +/, " "))
end