Module: Twilic::Core::Codec

Defined in:
lib/twilic/core/codec.rb

Constant Summary collapse

MAX_U64 = 
0xFFFFFFFFFFFFFFFF
MAX_I64 = 
0x7FFFFFFFFFFFFFFF
MIN_I64 = 
-0x8000000000000000
SIMPLE8B_SLOTS = 
[
  { count: 60, width: 1 },
  { count: 30, width: 2 },
  { count: 20, width: 3 },
  { count: 15, width: 4 },
  { count: 12, width: 5 },
  { count: 10, width: 6 },
  { count: 8, width: 7 },
  { count: 7, width: 8 },
  { count: 6, width: 10 },
  { count: 5, width: 12 },
  { count: 4, width: 15 },
  { count: 3, width: 20 },
  { count: 2, width: 30 },
  { count: 1, width: 60 }
].freeze

Class Method Summary collapse

Class Method Details

.bit_width(v) ⇒ Object 



721
722
723
724
725
726
 
# File 'lib/twilic/core/codec.rb', line 721

def bit_width(v)
  v &= MAX_U64
  return 1 if v.zero?

  64 - leading_zeros64(v)
end

.checked_add_i64(a, b) ⇒ Object 



733
734
735
736
737
738
739
 
# File 'lib/twilic/core/codec.rb', line 733

def checked_add_i64(a, b)
  sum = a + b
  return [0, false] if (b.positive? && sum < a) || (b.negative? && sum > a)
  return [0, false] if sum < MIN_I64 || sum > MAX_I64

  [sum, true]
end

.checked_add_u64(a, b) ⇒ Object 



728
729
730
731
 
# File 'lib/twilic/core/codec.rb', line 728

def checked_add_u64(a, b)
  sum = a + b
  [sum & MAX_U64, sum <= MAX_U64]
end

.decode_f64_vector(reader, codec) ⇒ Object 



187
188
189
190
191
192
193
194
195
196
 
# File 'lib/twilic/core/codec.rb', line 187

def decode_f64_vector(reader, codec)
  return decode_xor_float(reader) if codec == Model::VectorCodec::XOR_FLOAT

  length = reader.read_varuint
  out = []
  length.times do
    out << Wire.read_f64_le(reader)
  end
  out
end

.decode_i64_delta_delta(reader) ⇒ Object 



635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
 
# File 'lib/twilic/core/codec.rb', line 635

def decode_i64_delta_delta(reader)
  length = reader.read_varuint
  return [] if length.zero?

  first_encoded = reader.read_varuint
  first = Wire.decode_zigzag(first_encoded)
  return [first] if length == 1

  first_delta_encoded = reader.read_varuint
  first_delta = Wire.decode_zigzag(first_delta_encoded)
  dd = decode_i64_direct_bitpack(reader)
  raise Errors.invalid_data("delta-delta length") if dd.length != length - 2

  out = [first]
  prev = first
  second, ok = checked_add_i64(prev, first_delta)
  raise Errors.invalid_data("delta-delta overflow") unless ok

  out << second
  prev = second
  prev_delta = first_delta
  dd.each do |ddv|
    d, ok = checked_add_i64(prev_delta, ddv)
    raise Errors.invalid_data("delta-delta overflow") unless ok

    nxt, ok = checked_add_i64(prev, d)
    raise Errors.invalid_data("delta-delta overflow") unless ok

    out << nxt
    prev = nxt
    prev_delta = d
  end
  out
end

.decode_i64_direct_bitpack(reader) ⇒ Object 



605
606
607
608
609
610
611
612
613
614
 
# File 'lib/twilic/core/codec.rb', line 605

def decode_i64_direct_bitpack(reader)
  length = reader.read_varuint
  width = reader.read_u8
  return [] if length.zero?

  raise Errors.invalid_data("bitpack width") if width.zero? || width > 64

  encoded = unpack_u64_values(reader, length, width)
  encoded.map { |v| Wire.decode_zigzag(v) }
end

.decode_i64_patched_for(reader) ⇒ Object 



509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
 
# File 'lib/twilic/core/codec.rb', line 509

def decode_i64_patched_for(reader)
  length = reader.read_varuint
  return [] if length.zero?

  base_encoded = reader.read_varuint
  base = Wire.decode_zigzag(base_encoded)
  reader.read_u8
  values = []
  length.times do
    v = reader.read_varuint
    values << u64_to_i64(v)
  end
  patch_count = reader.read_varuint
  patch_count.times do
    pos = reader.read_varuint
    patch = reader.read_varuint
    values[pos] = u64_to_i64(patch) if pos < values.length
  end
  values.map { |v| v + base }
end

.decode_i64_plain(reader) ⇒ Object 



269
270
271
272
273
274
275
276
277
 
# File 'lib/twilic/core/codec.rb', line 269

def decode_i64_plain(reader)
  length = reader.read_varuint
  out = []
  length.times do
    v = reader.read_varuint
    out << Wire.decode_zigzag(v)
  end
  out
end

.decode_i64_rle(reader) ⇒ Object 



452
453
454
455
456
457
458
459
460
461
462
 
# File 'lib/twilic/core/codec.rb', line 452

def decode_i64_rle(reader)
  runs_len = reader.read_varuint
  out = []
  runs_len.times do
    value_encoded = reader.read_varuint
    value = Wire.decode_zigzag(value_encoded)
    count = reader.read_varuint
    count.times { out << value }
  end
  out
end

.decode_i64_simple8b(reader) ⇒ Object 



284
285
286
287
 
# File 'lib/twilic/core/codec.rb', line 284

def decode_i64_simple8b(reader)
  encoded = decode_u64_simple8b_inner(reader)
  encoded.map { |v| Wire.decode_zigzag(v) }
end

.decode_i64_vector(reader, codec) ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/twilic/core/codec.rb', line 79

def decode_i64_vector(reader, codec)
  case codec
  when Model::VectorCodec::RLE
    decode_i64_rle(reader)
  when Model::VectorCodec::DIRECT_BITPACK
    decode_i64_direct_bitpack(reader)
  when Model::VectorCodec::DELTA_BITPACK
    values = decode_i64_direct_bitpack(reader)
    undelta(values)
  when Model::VectorCodec::FOR_BITPACK
    encoded_min = reader.read_varuint
    min_value = Wire.decode_zigzag(encoded_min)
    return [] if reader.eof?

    shifted = decode_i64_direct_bitpack(reader)
    shifted.map { |v| v + min_value }
  when Model::VectorCodec::DELTA_FOR_BITPACK
    encoded_min = reader.read_varuint
    min_value = Wire.decode_zigzag(encoded_min)
    return [] if reader.eof?

    shifted = decode_i64_direct_bitpack(reader)
    deltas = shifted.map { |v| v + min_value }
    undelta(deltas)
  when Model::VectorCodec::DELTA_DELTA_BITPACK
    decode_i64_delta_delta(reader)
  when Model::VectorCodec::PATCHED_FOR
    decode_i64_patched_for(reader)
  when Model::VectorCodec::SIMPLE8B
    decode_i64_simple8b(reader)
  when Model::VectorCodec::PLAIN, Model::VectorCodec::DICTIONARY, Model::VectorCodec::STRING_REF,
       Model::VectorCodec::PREFIX_DELTA, Model::VectorCodec::XOR_FLOAT
    decode_i64_plain(reader)
  else
    raise Errors.invalid_data("unsupported vector codec")
  end
end

.decode_u64_direct_bitpack(reader) ⇒ Object 



254
255
256
257
258
259
260
261
262
 
# File 'lib/twilic/core/codec.rb', line 254

def decode_u64_direct_bitpack(reader)
  length = reader.read_varuint
  width = reader.read_u8
  return [] if length.zero?

  raise Errors.invalid_data("bitpack width") if width.zero? || width > 64

  unpack_u64_values(reader, length, width)
end

.decode_u64_plain(reader) ⇒ Object 



203
204
205
206
207
208
209
210
 
# File 'lib/twilic/core/codec.rb', line 203

def decode_u64_plain(reader)
  length = reader.read_varuint
  out = []
  length.times do
    out << reader.read_varuint
  end
  out
end

.decode_u64_rle(reader) ⇒ Object 



228
229
230
231
232
233
234
235
236
237
 
# File 'lib/twilic/core/codec.rb', line 228

def decode_u64_rle(reader)
  runs_len = reader.read_varuint
  out = []
  runs_len.times do
    value = reader.read_varuint
    count = reader.read_varuint
    count.times { out << value }
  end
  out
end

.decode_u64_simple8b(reader) ⇒ Object 



293
294
295
 
# File 'lib/twilic/core/codec.rb', line 293

def decode_u64_simple8b(reader)
  decode_u64_simple8b_inner(reader)
end

.decode_u64_simple8b_inner(reader) ⇒ Object 



360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
 
# File 'lib/twilic/core/codec.rb', line 360

def decode_u64_simple8b_inner(reader)
  length = reader.read_varuint
  return [] if length.zero?

  mode = reader.read_u8
  if mode.zero?
    out = []
    length.times do
      out << reader.read_varuint
    end
    return out
  end
  raise Errors.invalid_data("simple8b mode") unless mode == 1

  out = []
  while out.length < length
    packed = Wire.read_u64_le(reader)
    selector = packed >> 60
    payload = packed & ((1 << 60) - 1)
    if selector == 0 || selector == 1
      count = selector == 1 ? 120 : 240
      remain = length - out.length
      limit = remain < count ? remain : count
      limit.times { out << 0 }
    elsif selector >= 2 && selector <= 15
      if selector == 15
        count = 1
        width = 60
      else
        slot = SIMPLE8B_SLOTS[selector - 2]
        count = slot[:count]
        width = slot[:width]
      end
      mask = (1 << width) - 1
      shift = 0
      remain = length - out.length
      limit = remain < count ? remain : count
      limit.times do
        out << ((payload >> shift) & mask)
        shift += width
      end
    else
      raise Errors.invalid_data("simple8b selector")
    end
  end
  out
end

.decode_u64_vector(reader, codec) ⇒ Object 



146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/twilic/core/codec.rb', line 146

def decode_u64_vector(reader, codec)
  case codec
  when Model::VectorCodec::RLE
    decode_u64_rle(reader)
  when Model::VectorCodec::DIRECT_BITPACK
    decode_u64_direct_bitpack(reader)
  when Model::VectorCodec::FOR_BITPACK
    min_value = reader.read_varuint
    return [] if reader.eof?

    shifted = decode_u64_direct_bitpack(reader)
    out = []
    shifted.each do |v|
      sum, ok = checked_add_u64(v, min_value)
      raise Errors.invalid_data("u64 FOR overflow") unless ok

      out << sum
    end
    out
  when Model::VectorCodec::PLAIN
    decode_u64_plain(reader)
  when Model::VectorCodec::SIMPLE8B
    decode_u64_simple8b(reader)
  when Model::VectorCodec::DICTIONARY, Model::VectorCodec::STRING_REF, Model::VectorCodec::PREFIX_DELTA,
       Model::VectorCodec::XOR_FLOAT, Model::VectorCodec::DELTA_BITPACK, Model::VectorCodec::DELTA_FOR_BITPACK,
       Model::VectorCodec::DELTA_DELTA_BITPACK, Model::VectorCodec::PATCHED_FOR
    decode_u64_plain(reader)
  else
    raise Errors.invalid_data("unsupported vector codec")
  end
end

.decode_xor_float(reader) ⇒ Object 



561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
 
# File 'lib/twilic/core/codec.rb', line 561

def decode_xor_float(reader)
  length = reader.read_varuint
  return [] if length.zero?

  first_bits = Wire.read_u64_le(reader)
  out = [u64_to_f64(first_bits)]
  prev = first_bits
  (length - 1).times do
    flag = reader.read_u8
    bits_value = prev
    unless flag.zero?
      leading = reader.read_varuint
      trailing = reader.read_varuint
      width = reader.read_varuint
      payload = reader.read_varuint
      raise Errors.invalid_data("xor-float bit widths") if leading + trailing + width > 64

      x = width == 64 ? payload : (payload << trailing)
      bits_value = prev ^ x
    end
    out << u64_to_f64(bits_value)
    prev = bits_value
  end
  out
end

.delta(values) ⇒ Object 



408
409
410
411
412
413
414
415
416
 
# File 'lib/twilic/core/codec.rb', line 408

def delta(values)
  out = []
  prev = 0
  values.each_with_index do |value, i|
    out << (i.zero? ? value : (value - prev))
    prev = value
  end
  out
end

.encode_f64_vector(values, codec, out) ⇒ Object 



178
179
180
181
182
183
184
185
 
# File 'lib/twilic/core/codec.rb', line 178

def encode_f64_vector(values, codec, out)
  if codec == Model::VectorCodec::XOR_FLOAT
    encode_xor_float(values, out)
    return
  end
  Wire.encode_varuint(values.length, out)
  values.each { |v| Wire.append_f64_le(out, v) }
end

.encode_i64_delta_delta(values, out) ⇒ Object 



616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
 
# File 'lib/twilic/core/codec.rb', line 616

def encode_i64_delta_delta(values, out)
  Wire.encode_varuint(values.length, out)
  return if values.empty?

  Wire.encode_varuint(Wire.encode_zigzag(values[0]), out)
  return if values.length == 1

  d1 = values[1] - values[0]
  Wire.encode_varuint(Wire.encode_zigzag(d1), out)
  dd = []
  prev_delta = d1
  (1...(values.length - 1)).each do |i|
    d = values[i + 1] - values[i]
    dd << (d - prev_delta)
    prev_delta = d
  end
  encode_i64_direct_bitpack(dd, out)
end

.encode_i64_direct_bitpack(values, out) ⇒ Object 



587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
 
# File 'lib/twilic/core/codec.rb', line 587

def encode_i64_direct_bitpack(values, out)
  Wire.encode_varuint(values.length, out)
  if values.empty?
    out << 0.chr
    return
  end
  encoded = []
  width = 1
  values.each do |v|
    enc = Wire.encode_zigzag(v)
    encoded << enc
    bw = bit_width(enc)
    width = bw if bw > width
  end
  out << width.chr
  pack_u64_values(encoded, width, out)
end

.encode_i64_patched_for(values, out) ⇒ Object 



464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
 
# File 'lib/twilic/core/codec.rb', line 464

def encode_i64_patched_for(values, out)
  if values.empty?
    Wire.encode_varuint(0, out)
    return
  end
  base = values[0]
  values[1..].each do |v|
    base = v if v < base
  end
  shifted = values.map { |v| v - base }
  Wire.encode_varuint(shifted.length, out)
  Wire.encode_varuint(Wire.encode_zigzag(base), out)

  max_value = 0
  shifted.each { |value| max_value = value if value > max_value }
  bw = bit_width(max_value & MAX_U64)
  base_width = bw > 2 ? bw - 2 : 0
  out << base_width.chr

  patch_positions = []
  main_values = []
  shifted.each_with_index do |value, idx|
    if bit_width(value & MAX_U64) > base_width
      patch_positions << { pos: idx, value: value }
      main = 0
      if base_width.positive?
        mask = (1 << base_width) - 1
        main = value & mask
        main = 0 if main.negative?
      end
      main_values << main
    else
      main_values << value
    end
  end
  main_values.each do |value|
    Wire.encode_varuint(value & MAX_U64, out)
  end
  Wire.encode_varuint(patch_positions.length, out)
  patch_positions.each do |patch|
    Wire.encode_varuint(patch[:pos], out)
    Wire.encode_varuint(patch[:value] & MAX_U64, out)
  end
end

.encode_i64_plain(values, out) ⇒ Object 



264
265
266
267
 
# File 'lib/twilic/core/codec.rb', line 264

def encode_i64_plain(values, out)
  Wire.encode_varuint(values.length, out)
  values.each { |value| Wire.encode_varuint(Wire.encode_zigzag(value), out) }
end

.encode_i64_rle(values, out) ⇒ Object 



436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
 
# File 'lib/twilic/core/codec.rb', line 436

def encode_i64_rle(values, out)
  runs = []
  values.each do |value|
    if !runs.empty? && runs[-1][:value] == value
      runs[-1][:count] += 1
    else
      runs << { value: value, count: 1 }
    end
  end
  Wire.encode_varuint(runs.length, out)
  runs.each do |run|
    Wire.encode_varuint(Wire.encode_zigzag(run[:value]), out)
    Wire.encode_varuint(run[:count], out)
  end
end

.encode_i64_simple8b(values, out) ⇒ Object 



279
280
281
282
 
# File 'lib/twilic/core/codec.rb', line 279

def encode_i64_simple8b(values, out)
  encoded = values.map { |v| Wire.encode_zigzag(v) }
  encode_u64_simple8b_inner(encoded, out)
end

.encode_i64_vector(values, codec, out) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/twilic/core/codec.rb', line 33

def encode_i64_vector(values, codec, out)
  case codec
  when Model::VectorCodec::RLE
    encode_i64_rle(values, out)
  when Model::VectorCodec::DIRECT_BITPACK
    encode_i64_direct_bitpack(values, out)
  when Model::VectorCodec::DELTA_BITPACK
    deltas = delta(values)
    encode_i64_direct_bitpack(deltas, out)
  when Model::VectorCodec::FOR_BITPACK
    if values.empty?
      Wire.encode_varuint(0, out)
      return
    end
    min_value = values[0]
    values[1..].each do |v|
      min_value = v if v < min_value
    end
    Wire.encode_varuint(Wire.encode_zigzag(min_value), out)
    shifted = values.map { |v| v - min_value }
    encode_i64_direct_bitpack(shifted, out)
  when Model::VectorCodec::DELTA_FOR_BITPACK
    deltas = delta(values)
    if deltas.empty?
      Wire.encode_varuint(0, out)
      return
    end
    min_value = deltas[0]
    deltas[1..].each do |v|
      min_value = v if v < min_value
    end
    Wire.encode_varuint(Wire.encode_zigzag(min_value), out)
    shifted = deltas.map { |v| v - min_value }
    encode_i64_direct_bitpack(shifted, out)
  when Model::VectorCodec::DELTA_DELTA_BITPACK
    encode_i64_delta_delta(values, out)
  when Model::VectorCodec::PATCHED_FOR
    encode_i64_patched_for(values, out)
  when Model::VectorCodec::SIMPLE8B
    encode_i64_simple8b(values, out)
  when Model::VectorCodec::PLAIN, Model::VectorCodec::DICTIONARY, Model::VectorCodec::STRING_REF,
       Model::VectorCodec::PREFIX_DELTA, Model::VectorCodec::XOR_FLOAT
    encode_i64_plain(values, out)
  end
end

.encode_u64_direct_bitpack(values, out) ⇒ Object 



239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/twilic/core/codec.rb', line 239

def encode_u64_direct_bitpack(values, out)
  Wire.encode_varuint(values.length, out)
  if values.empty?
    out << 0.chr
    return
  end
  width = 1
  values.each do |v|
    bw = bit_width(v)
    width = bw if bw > width
  end
  out << width.chr
  pack_u64_values(values, width, out)
end

.encode_u64_plain(values, out) ⇒ Object 



198
199
200
201
 
# File 'lib/twilic/core/codec.rb', line 198

def encode_u64_plain(values, out)
  Wire.encode_varuint(values.length, out)
  values.each { |value| Wire.encode_varuint(value, out) }
end

.encode_u64_rle(values, out) ⇒ Object 



212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
 
# File 'lib/twilic/core/codec.rb', line 212

def encode_u64_rle(values, out)
  runs = []
  values.each do |value|
    if !runs.empty? && runs[-1][:value] == value
      runs[-1][:count] += 1
    else
      runs << { value: value, count: 1 }
    end
  end
  Wire.encode_varuint(runs.length, out)
  runs.each do |run|
    Wire.encode_varuint(run[:value], out)
    Wire.encode_varuint(run[:count], out)
  end
end

.encode_u64_simple8b(values, out) ⇒ Object 



289
290
291
 
# File 'lib/twilic/core/codec.rb', line 289

def encode_u64_simple8b(values, out)
  encode_u64_simple8b_inner(values, out)
end

.encode_u64_simple8b_inner(values, out) ⇒ Object 



297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
 
# File 'lib/twilic/core/codec.rb', line 297

def encode_u64_simple8b_inner(values, out)
  Wire.encode_varuint(values.length, out)
  return if values.empty?

  max_value = 0
  values.each { |v| max_value = v if v > max_value }
  if max_value > ((1 << 60) - 1)
    out << 0.chr
    values.each { |value| Wire.encode_varuint(value, out) }
    return
  end

  out << 1.chr
  idx = 0
  while idx < values.length
    zero_run = 0
    while idx + zero_run < values.length && values[idx + zero_run].zero? && zero_run < 240
      zero_run += 1
    end
    if zero_run >= 120
      take = zero_run >= 240 ? 240 : 120
      word = (take == 240 ? 0 : (1 << 60))
      Wire.append_u64_le(out, word)
      idx += take
      next
    end

    packed = false
    SIMPLE8B_SLOTS.each_with_index do |slot, selector_idx|
      next if idx + slot[:count] > values.length

      max_encodable = (1 << slot[:width]) - 1
      all_fit = true
      values[idx, slot[:count]].each do |value|
        if value > max_encodable
          all_fit = false
          break
        end
      end
      next unless all_fit

      selector = selector_idx + 2
      payload = 0
      shift = 0
      values[idx, slot[:count]].each do |value|
        payload |= (value << shift)
        shift += slot[:width]
      end
      word = (selector << 60) | payload
      Wire.append_u64_le(out, word & MAX_U64)
      idx += slot[:count]
      packed = true
      break
    end
    next if packed

    selector = 15
    word = (selector << 60) | (values[idx] & ((1 << 60) - 1))
    Wire.append_u64_le(out, word & MAX_U64)
    idx += 1
  end
end

.encode_u64_vector(values, codec, out) ⇒ Object 



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/twilic/core/codec.rb', line 117

def encode_u64_vector(values, codec, out)
  case codec
  when Model::VectorCodec::RLE
    encode_u64_rle(values, out)
  when Model::VectorCodec::DIRECT_BITPACK
    encode_u64_direct_bitpack(values, out)
  when Model::VectorCodec::FOR_BITPACK
    if values.empty?
      Wire.encode_varuint(0, out)
      return
    end
    min_value = values[0]
    values[1..].each do |v|
      min_value = v if v < min_value
    end
    Wire.encode_varuint(min_value, out)
    shifted = values.map { |v| v - min_value }
    encode_u64_direct_bitpack(shifted, out)
  when Model::VectorCodec::PLAIN
    encode_u64_plain(values, out)
  when Model::VectorCodec::SIMPLE8B
    encode_u64_simple8b(values, out)
  when Model::VectorCodec::DICTIONARY, Model::VectorCodec::STRING_REF, Model::VectorCodec::PREFIX_DELTA,
       Model::VectorCodec::XOR_FLOAT, Model::VectorCodec::DELTA_BITPACK, Model::VectorCodec::DELTA_FOR_BITPACK,
       Model::VectorCodec::DELTA_DELTA_BITPACK, Model::VectorCodec::PATCHED_FOR
    encode_u64_plain(values, out)
  end
end

.encode_xor_float(values, out) ⇒ Object 



530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
 
# File 'lib/twilic/core/codec.rb', line 530

def encode_xor_float(values, out)
  Wire.encode_varuint(values.length, out)
  return if values.empty?

  first_bits = f64_to_u64(values[0])
  Wire.append_u64_le(out, first_bits)
  prev = first_bits
  values[1..].each do |value|
    bits_value = f64_to_u64(value)
    x = prev ^ bits_value
    if x.zero?
      out << 0.chr
    else
      out << 1.chr
      leading = leading_zeros64(x)
      trailing = trailing_zeros64(x)
      width = 64 - (leading + trailing)
      Wire.encode_varuint(leading, out)
      Wire.encode_varuint(trailing, out)
      Wire.encode_varuint(width, out)
      payload = if width == 64
                  x
                else
                  (x >> trailing) & ((1 << width) - 1)
                end
      Wire.encode_varuint(payload, out)
    end
    prev = bits_value
  end
end

.f64_to_u64(value) ⇒ Object 



745
746
747
 
# File 'lib/twilic/core/codec.rb', line 745

def f64_to_u64(value)
  [value].pack("E").unpack1("Q<")
end

.leading_zeros64(v) ⇒ Object 



753
754
755
756
757
 
# File 'lib/twilic/core/codec.rb', line 753

def leading_zeros64(v)
  return 64 if v.zero?

  64 - v.bit_length
end

.pack_u64_values(values, width, out) ⇒ Object 



670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
 
# File 'lib/twilic/core/codec.rb', line 670

def pack_u64_values(values, width, out)
  total_bits = values.length * width
  byte_len = (total_bits + 7) / 8
  bytes = Array.new(byte_len, 0)
  bit_pos = 0
  values.each do |value|
    written = 0
    while written < width
      byte_idx = bit_pos / 8
      bit_off = bit_pos % 8
      room = 8 - bit_off
      take = width - written
      take = room if take > room
      mask = (1 << take) - 1
      part = (value >> written) & mask
      bytes[byte_idx] |= (part << bit_off)
      bit_pos += take
      written += take
    end
  end
  out << bytes.pack("C*")
end

.trailing_zeros64(v) ⇒ Object 



759
760
761
762
763
 
# File 'lib/twilic/core/codec.rb', line 759

def trailing_zeros64(v)
  return 64 if v.zero?

  (v & -v).bit_length - 1
end

.u64_to_f64(bits) ⇒ Object 



749
750
751
 
# File 'lib/twilic/core/codec.rb', line 749

def u64_to_f64(bits)
  [bits].pack("Q<").unpack1("E")
end

.u64_to_i64(v) ⇒ Object 



741
742
743
 
# File 'lib/twilic/core/codec.rb', line 741

def u64_to_i64(v)
  (v & (1 << 63)).zero? ? v : (v - (1 << 64))
end

.undelta(values) ⇒ Object 



418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
 
# File 'lib/twilic/core/codec.rb', line 418

def undelta(values)
  out = []
  prev = 0
  values.each_with_index do |value, i|
    if i.zero?
      out << value
      prev = value
      next
    end
    next_value, ok = checked_add_i64(prev, value)
    raise Errors.invalid_data("delta overflow") unless ok

    out << next_value
    prev = next_value
  end
  out
end

.unpack_u64_values(reader, length, width) ⇒ Object 



693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
 
# File 'lib/twilic/core/codec.rb', line 693

def unpack_u64_values(reader, length, width)
  total_bits = length * width
  byte_len = (total_bits + 7) / 8
  bytes = reader.read_exact(byte_len)
  out = []
  bit_pos = 0
  length.times do
    value = 0
    written = 0
    while written < width
      byte_idx = bit_pos / 8
      raise Errors.invalid_data("bitpack underflow") if byte_idx >= bytes.bytesize

      bit_off = bit_pos % 8
      room = 8 - bit_off
      take = width - written
      take = room if take > room
      mask = (1 << take) - 1
      part = (bytes.getbyte(byte_idx) >> bit_off) & mask
      value |= (part << written)
      bit_pos += take
      written += take
    end
    out << value
  end
  out
end