Module: WTF8::Codec

Defined in:
lib/wtf8/codec.rb,
sig/wtf8/codec.rbs

Constant Summary collapse

MAX_CODE_POINT =

Returns:

  • (::Integer) 
0x10FFFF
REPLACEMENT_CHARACTER =

Returns:

  • (::String) 
"\u{FFFD}"

Class Method Summary collapse

Class Method Details

.concat(left, right) ⇒ ::String

: (::String, ::String) -> ::String

Parameters:

  • (::String)
  • (::String)

Returns:

  • (::String) 


143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/wtf8/codec.rb', line 143

def self.concat(left, right)
  left = left.b
  right = right.b

  return left + right unless ends_with_lead_surrogate?(left) && starts_with_trail_surrogate?(right)

  lead_bytes = left.byteslice(-3, 3) #: ::String
  trail_bytes = right.byteslice(0, 3) #: ::String
  head = left.byteslice(0, left.bytesize - 3) #: ::String
  tail = right.byteslice(3, right.bytesize - 3) #: ::String

  lead = lead_bytes.unpack1("U") #: Integer
  trail = trail_bytes.unpack1("U") #: Integer

  head + [Surrogates.combine(lead, trail)].pack("U").b + tail
end

.decode(bytes) ⇒ Array[Integer]

: (::String) -> Array

Parameters:

  • (::String)

Returns:

  • (Array[Integer]) 


9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/wtf8/codec.rb', line 9

def self.decode(bytes)
  code_points = unpack(bytes)

  code_points.each do |code_point|
    next unless code_point > MAX_CODE_POINT

    raise InvalidCodePointError, format("U+%04X is above U+10FFFF", code_point)
  end

  code_points.each_cons(2) do |pair|
    lead = pair[0]
    trail = pair[1]

    next unless Surrogates.lead?(lead) && Surrogates.trail?(trail)

    raise InvalidByteSequenceError, format("U+%<lead>04X U+%<trail>04X is an encoded surrogate pair, which WTF-8 writes as one four-byte sequence", lead: lead, trail: trail)
  end

  code_points
end

.encode(code_points) ⇒ ::String

: (Array) -> ::String

Parameters:

  • (Array[Integer])

Returns:

  • (::String) 


38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/wtf8/codec.rb', line 38

def self.encode(code_points)
  code_points.each do |code_point|
    next if code_point.is_a?(Integer) && code_point >= 0 && code_point <= MAX_CODE_POINT

    raise InvalidCodePointError, "#{code_point.inspect} is not a code point in U+0000..U+10FFFF"
  end

  code_points.each_cons(2) do |pair|
    next unless Surrogates.lead?(pair[0]) && Surrogates.trail?(pair[1])

    raise InvalidCodePointError, "a surrogate pair has to be combined before it is encoded, which from_utf16 does"
  end

  code_points.each_with_object(+"".b) { |code_point, bytes| bytes << [code_point].pack("U").b }
end

.ends_with_lead_surrogate?(bytes) ⇒ Boolean

: (::String) -> bool

Parameters:

  • (::String)

Returns:

  • (Boolean) 


161
162
163
 
# File 'lib/wtf8/codec.rb', line 161

def self.ends_with_lead_surrogate?(bytes)
  bytes.bytesize >= 3 && bytes.getbyte(-3) == Surrogates::PREFIX_BYTE && Surrogates::LEAD_SECOND_BYTE.cover?(bytes.getbyte(-2))
end

.from_utf16(units) ⇒ ::String

: (Array) -> ::String

Parameters:

  • (Array[Integer])

Returns:

  • (::String) 


88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/wtf8/codec.rb', line 88

def self.from_utf16(units)
  bytes = +"".b
  index = 0

  while index < units.length
    unit = units[index]

    unless unit.is_a?(Integer) && unit >= 0 && unit <= 0xFFFF
      raise InvalidCodePointError, "#{unit.inspect} is not a UTF-16 code unit"
    end

    following = units[index + 1]

    if Surrogates.lead?(unit) && following && Surrogates.trail?(following)
      bytes << [Surrogates.combine(unit, following)].pack("U").b
      index += 2
    else
      bytes << [unit].pack("U").b
      index += 1
    end
  end

  bytes
end

.from_utf16be(bytes) ⇒ ::String

: (::String) -> ::String

Parameters:

  • (::String)

Returns:

  • (::String) 


126
127
128
129
130
 
# File 'lib/wtf8/codec.rb', line 126

def self.from_utf16be(bytes)
  units = bytes.b.unpack("n*") #: Array[Integer]

  from_utf16(units)
end

.from_utf16le(bytes) ⇒ ::String

: (::String) -> ::String

Parameters:

  • (::String)

Returns:

  • (::String) 


119
120
121
122
123
 
# File 'lib/wtf8/codec.rb', line 119

def self.from_utf16le(bytes)
  units = bytes.b.unpack("v*") #: Array[Integer]

  from_utf16(units)
end

.from_utf8(string) ⇒ ::String

: (::String) -> ::String

Parameters:

  • (::String)

Returns:

  • (::String) 


68
69
70
71
72
73
74
 
# File 'lib/wtf8/codec.rb', line 68

def self.from_utf8(string)
  string = string.encode(Encoding::UTF_8) unless string.encoding == Encoding::UTF_8 || string.ascii_only?

  raise InvalidByteSequenceError, "not valid UTF-8" unless string.valid_encoding?

  string.b
end

.starts_with_trail_surrogate?(bytes) ⇒ Boolean

: (::String) -> bool

Parameters:

  • (::String)

Returns:

  • (Boolean) 


166
167
168
 
# File 'lib/wtf8/codec.rb', line 166

def self.starts_with_trail_surrogate?(bytes)
  bytes.bytesize >= 3 && bytes.getbyte(0) == Surrogates::PREFIX_BYTE && Surrogates::TRAIL_SECOND_BYTE.cover?(bytes.getbyte(1))
end

.to_utf16(bytes) ⇒ Array[Integer]

: (::String) -> Array

Parameters:

  • (::String)

Returns:

  • (Array[Integer]) 


114
115
116
 
# File 'lib/wtf8/codec.rb', line 114

def self.to_utf16(bytes)
  decode(bytes).flat_map { |code_point| code_point > 0xFFFF ? Surrogates.split(code_point) : code_point }
end

.to_utf16be(bytes) ⇒ ::String

: (::String) -> ::String

Parameters:

  • (::String)

Returns:

  • (::String) 


138
139
140
 
# File 'lib/wtf8/codec.rb', line 138

def self.to_utf16be(bytes)
  to_utf16(bytes).pack("n*")
end

.to_utf16le(bytes) ⇒ ::String

: (::String) -> ::String

Parameters:

  • (::String)

Returns:

  • (::String) 


133
134
135
 
# File 'lib/wtf8/codec.rb', line 133

def self.to_utf16le(bytes)
  to_utf16(bytes).pack("v*")
end

.to_utf8(bytes, replacement: REPLACEMENT_CHARACTER) ⇒ ::String

: (::String, ?replacement: ::String) -> ::String

Parameters:

  • (::String)
  • replacement: (::String) (defaults to: REPLACEMENT_CHARACTER)

Returns:

  • (::String) 


77
78
79
80
81
82
83
84
85
 
# File 'lib/wtf8/codec.rb', line 77

def self.to_utf8(bytes, replacement: REPLACEMENT_CHARACTER)
  utf8 = +""

  decode(bytes).each do |code_point|
    utf8 << (Surrogates.surrogate?(code_point) ? replacement : [code_point].pack("U"))
  end

  utf8.force_encoding(Encoding::UTF_8)
end

.unpack(bytes) ⇒ Array[Integer]

: (::String) -> Array

Parameters:

  • (::String)

Returns:

  • (Array[Integer]) 


31
32
33
34
35
 
# File 'lib/wtf8/codec.rb', line 31

def self.unpack(bytes)
  bytes.b.unpack("U*") #: Array[Integer]
rescue ArgumentError => e
  raise InvalidByteSequenceError, e.message
end

.valid?(bytes) ⇒ Boolean

: (::String) -> bool

Parameters:

  • (::String)

Returns:

  • (Boolean) 


55
56
57
58
59
60
 
# File 'lib/wtf8/codec.rb', line 55

def self.valid?(bytes)
  decode(bytes)
  true
rescue Error
  false
end

.well_formed?(bytes) ⇒ Boolean

: (::String) -> bool

Parameters:

  • (::String)

Returns:

  • (Boolean) 


63
64
65
 
# File 'lib/wtf8/codec.rb', line 63

def self.well_formed?(bytes)
  bytes.b.dup.force_encoding(Encoding::UTF_8).valid_encoding?
end