Module: Flexr::Unicode::Utf8Splitter

Defined in:
lib/flexr/unicode/utf8_splitter.rb

Class Method Summary collapse

Class Method Details

.decode(bytes) ⇒ Object 



98
99
100
101
102
103
104
105
106
 
# File 'lib/flexr/unicode/utf8_splitter.rb', line 98

def decode(bytes)
  return bytes.first if bytes.length == 1
  case bytes.length
  when 2 then ((bytes[0] & 0x1f) << 6) | (bytes[1] & 0x3f)
  when 3 then ((bytes[0] & 0x0f) << 12) | ((bytes[1] & 0x3f) << 6) | (bytes[2] & 0x3f)
  when 4 then ((bytes[0] & 0x07) << 18) | ((bytes[1] & 0x3f) << 12) |
    ((bytes[2] & 0x3f) << 6) | (bytes[3] & 0x3f)
  end
end

.encoded(codepoint) ⇒ Object 



64
65
66
 
# File 'lib/flexr/unicode/utf8_splitter.rb', line 64

def encoded(codepoint)
  [codepoint].pack("U").bytes
end

.prefix_bounds(prefix, length) ⇒ Object 



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/flexr/unicode/utf8_splitter.rb', line 68

def prefix_bounds(prefix, length)
  return [nil, nil] if prefix.empty?

  bytes = prefix + Array.new(length - prefix.length, 0x80)
  high = prefix + Array.new(length - prefix.length, 0xbf)
  if prefix.length == 1
    bytes[1] = 0xa0 if length == 3 && prefix.first == 0xe0
    high[1] = 0x9f if length == 3 && prefix.first == 0xed
    bytes[1] = 0x90 if length == 4 && prefix.first == 0xf0
    high[1] = 0x8f if length == 4 && prefix.first == 0xf4
  end
  return [nil, nil] unless valid_prefix?(bytes, length) && valid_prefix?(high, length)

  [decode(bytes), decode(high)]
end

.split(lo, hi) ⇒ Object

Raises:

  • (ArgumentError) 


8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/flexr/unicode/utf8_splitter.rb', line 8

def split(lo, hi)
  raise ArgumentError, "invalid codepoint range" if lo > hi || lo.negative? || hi > 0x10ffff
  return [] if lo.between?(0xd800, 0xdfff) && hi.between?(0xd800, 0xdfff)
  return [encoded(lo).map { |byte| [byte, byte] }] if lo == hi

  ranges = []
  [[0, 0x7f, 1], [0x80, 0x7ff, 2], [0x800, 0xffff, 3], [0x10000, 0x10ffff, 4]].each do |min, max, length|
    lower = [lo, min].max
    upper = [hi, max].min
    next if lower > upper

    walk(lower, upper, length, [], ranges)
  end
  ranges
end

.valid_prefix?(bytes, length) ⇒ Boolean

Returns:

  • (Boolean) 


84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/flexr/unicode/utf8_splitter.rb', line 84

def valid_prefix?(bytes, length)
  first = bytes.first
  expected = if first <= 0x7f then 1 elsif first.between?(0xc2, 0xdf) then 2
             elsif first.between?(0xe0, 0xef) then 3
             elsif first.between?(0xf0, 0xf4) then 4 end
  return false unless expected == length
  return false if bytes[1..].any? { |byte| !byte.between?(0x80, 0xbf) }
  return false if length == 3 && first == 0xe0 && bytes[1] < 0xa0
  return false if length == 3 && first == 0xed && bytes[1] > 0x9f
  return false if length == 4 && first == 0xf0 && bytes[1] < 0x90
  return false if length == 4 && first == 0xf4 && bytes[1] > 0x8f
  true
end

.walk(lo, hi, length, prefix, output) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/flexr/unicode/utf8_splitter.rb', line 24

def walk(lo, hi, length, prefix, output)
  if prefix.length == length
    output << prefix.map { |byte| [byte, byte] }
    return
  end

  if prefix.length == length - 1
    first_byte = prefix.empty? ? encoded(lo)[0] : 0x80
    last_byte = prefix.empty? ? encoded(hi)[0] : 0xbf
    group_start = nil
    first_byte.upto(last_byte) do |byte|
      min_cp, max_cp = prefix_bounds(prefix + [byte], length)
      allowed = min_cp && lo <= min_cp && max_cp <= hi
      if allowed
        group_start ||= byte
      elsif group_start
        output << (prefix.map { |value| [value, value] } + [[group_start, byte - 1]])
        group_start = nil
      end
    end
    output << (prefix.map { |value| [value, value] } + [[group_start, last_byte]]) if group_start
    return
  end

  first_byte = prefix.empty? ? encoded(lo)[0] : 0x80
  last_byte = prefix.empty? ? encoded(hi)[0] : 0xbf
  first_byte.upto(last_byte) do |byte|
    next_prefix = prefix + [byte]
    min_cp, max_cp = prefix_bounds(next_prefix, length)
    next if min_cp.nil? || max_cp < lo || min_cp > hi

    if lo <= min_cp && max_cp <= hi
      rest = length - next_prefix.length
      output << (next_prefix.map { |value| [value, value] } + Array.new(rest) { [0x80, 0xbf] })
    else
      walk(lo, hi, length, next_prefix, output)
    end
  end
end