Module: StaticEmbeddings::UnicodeTables

Defined in:
lib/static_embeddings/unicode_tables.rb

Defined Under Namespace

Classes: RangeAccumulator

Constant Summary collapse

MAP_SCAN_LIMIT = 
0x2FFFF
CODESPACE_LIMIT = 
0x10FFFF
SURROGATES = 
(0xD800..0xDFFF)
ENTRY_PACK = 
"V6"
RE_MN = 
/\A\p{Mn}\z/
RE_PUNCT = 
/\A\p{P}\z/
RE_CONTROL = 
/\A(?:\p{Cc}|\p{Cf}|\p{Co})\z/
RE_WHITESPACE = 
/\A(?:\p{Zs}|[\u0085\u2028\u2029])\z/

Class Method Summary collapse

Class Method Details

.build ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/static_embeddings/unicode_tables.rb', line 37

def build
  lower = []
  nfd = []
  mn = RangeAccumulator.new
  punct = RangeAccumulator.new
  control = RangeAccumulator.new
  whitespace = RangeAccumulator.new

  (0..CODESPACE_LIMIT).each do |cp|
    if SURROGATES.cover?(cp)
      control.add(cp)
      next
    end

    ch = begin
      cp.chr(Encoding::UTF_8)
    rescue RangeError
      next
    end

    mn.add(cp) if RE_MN.match?(ch)
    punct.add(cp) if RE_PUNCT.match?(ch)
    control.add(cp) if RE_CONTROL.match?(ch)
    whitespace.add(cp) if RE_WHITESPACE.match?(ch)

    next if cp > MAP_SCAN_LIMIT

    down = ch.downcase
    lower << [cp, down.codepoints] if down != ch

    decomposed = ch.unicode_normalize(:nfd)
    nfd << [cp, decomposed.codepoints] if decomposed != ch
  end

  { "lowercase" => lower, "NFD" => nfd }.each do |name, table|
    long = table.find { |(_, out)| out.length > 4 }
    next unless long

    raise "#{name} mapping for U+#{format('%04X', long[0])} needs #{long[1].length} slots"
  end

  {
    lower: lower,
    nfd: nfd,
    mn: mn.to_a,
    punct: punct.to_a,
    control: control.to_a,
    whitespace: whitespace.to_a
  }
end

.pack(tables) ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/static_embeddings/unicode_tables.rb', line 88

def pack(tables)
  blob = +""
  blob.force_encoding(Encoding::BINARY)
  blob << [
    tables[:lower].length,
    tables[:nfd].length,
    tables[:mn].length,
    tables[:punct].length,
    tables[:control].length,
    tables[:whitespace].length,
    0, 0
  ].pack("V8")

  %i[lower nfd].each do |key|
    tables[key].sort_by(&:first).each do |(cp, out)|
      padded = out + [0] * (4 - out.length)
      blob << ([cp, out.length] + padded).pack(ENTRY_PACK)
    end
  end

  %i[mn punct control whitespace].each do |key|
    tables[key].sort_by(&:first).each { |(lo, hi)| blob << [lo, hi].pack("V2") }
  end

  blob
end

.packed ⇒ Object 



115
116
117
 
# File 'lib/static_embeddings/unicode_tables.rb', line 115

def packed
  @packed ||= pack(build)
end

.reset! ⇒ Object 



119
120
121
 
# File 'lib/static_embeddings/unicode_tables.rb', line 119

def reset!
  @packed = nil
end

.source_stamp ⇒ Object 



123
124
125
 
# File 'lib/static_embeddings/unicode_tables.rb', line 123

def source_stamp
  "ruby-#{RUBY_VERSION}p#{RUBY_PATCHLEVEL}"
end