Module: Twilic::Core::Protocol

Defined in:
lib/twilic/core/protocol.rb,
lib/twilic/core/protocol.rb

Defined Under Namespace

Classes: SessionEncoder, TwilicCodec

Constant Summary collapse

TAG_NULL = 
0
TAG_BOOL_FALSE = 
1
TAG_BOOL_TRUE = 
2
TAG_I64 = 
3
TAG_U64 = 
4
TAG_F64 = 
5
TAG_STRING = 
6
TAG_BINARY = 
7
TAG_ARRAY = 
8
TAG_MAP = 
9

Class Method Summary collapse

Class Method Details

.column_null_strategy(values, present_bits) ⇒ Object 



1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
 
# File 'lib/twilic/core/protocol.rb', line 1827

def column_null_strategy(values, present_bits)
  null_count = 0
  values.each do |value|
    null_count += 1 if value.kind == Model::ValueKind::NULL
  end
  optional_count = values.length
  if null_count.zero?
    return [Model::NullStrategy::ALL_PRESENT_ELIDED, nil, false]
  end
  if null_count <= optional_count / 4
    inverted = Array.new(present_bits.length, false)
    present_bits.each_with_index do |bit, i|
      inverted[i] = !bit
    end
    return [Model::NullStrategy::INVERTED_PRESENCE_BITMAP, inverted, true]
  end
  [Model::NullStrategy::PRESENCE_BITMAP, present_bits.dup, true]
end

.columns_from_map_values(values) ⇒ Object 



1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
 
# File 'lib/twilic/core/protocol.rb', line 1854

def columns_from_map_values(values)
  return nil if values.empty?

  values.each do |value|
    return nil unless value.kind == Model::ValueKind::MAP
  end
  key_order = []
  key_index = {}
  column_values = []
  column_presence = []
  values.each_with_index do |row_value, row_idx|
    present = Array.new(key_order.length, false)
    row_value.map.each do |entry|
      key = entry.key
      entry_value = entry.value.clone_value
      col_idx = key_index[key]
      unless col_idx
        col_idx = key_order.length
        key_order << key
        key_index[key] = col_idx
        column_values << Array.new(row_idx)
        column_presence << Array.new(row_idx)
        present << false
      end
      column_values[col_idx] << entry_value
      column_presence[col_idx] << true
      present[col_idx] = true
    end
    key_order.each_index do |col_idx|
      next if present[col_idx]

      column_values[col_idx] << Model.null_value
      column_presence[col_idx] << false
    end
  end
  columns = Array.new(key_order.length)
  key_order.each_index do |field_id|
    col_values = column_values[field_id]
    present_bits = column_presence[field_id]
    null_strategy, presence, has_presence = column_null_strategy(col_values, present_bits)
    codec, tvd = infer_column_codec_and_values(strip_nulls(col_values))
    columns[field_id] = Model::Column.new(
      field_id: field_id,
      null_strategy: null_strategy,
      presence: presence,
      has_presence: has_presence,
      codec: codec,
      dictionary_id: nil,
      values: tvd
    )
  end
  columns
end

.delegate_helpers_to(klass) ⇒ Object 



2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
 
# File 'lib/twilic/core/protocol.rb', line 2023

def self.delegate_helpers_to(klass)
  ProtocolHelpers.singleton_methods(false).each do |name|
    next if klass.method_defined?(name)

    klass.define_method(name) do |*args, **kwargs, &block|
      ProtocolHelpers.send(name, *args, **kwargs, &block)
    end
  end
  singleton_methods(false).each do |name|
    next if klass.method_defined?(name)

    klass.define_method(name) do |*args, **kwargs, &block|
      Protocol.send(name, *args, **kwargs, &block)
    end
  end
end

.encoded_size(message) ⇒ Object 



1935
1936
1937
 
# File 'lib/twilic/core/protocol.rb', line 1935

def encoded_size(message)
  estimate_message_size(message)
end

.entries_to_map(entries, state) ⇒ Object 



1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
 
# File 'lib/twilic/core/protocol.rb', line 1966

def entries_to_map(entries, state)
  out = Array.new(entries.length)
  entries.each_with_index do |entry, i|
    key = key_ref_string(entry.key, state)
    out[i] = Model::MapEntry.new(key, entry.value.clone_value)
    _id, ok = state.key_table.get_id(key)
    state.key_table.register(key) unless ok
  end
  out
end

.has_uniform_micro_batch_shape(values) ⇒ Object 



1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
 
# File 'lib/twilic/core/protocol.rb', line 1908

def has_uniform_micro_batch_shape(values)
  return false if values.empty?
  return false if values[0].kind != Model::ValueKind::MAP

  keys = values[0].map.map(&:key)
  (1...values.length).each do |i|
    return false if values[i].kind != Model::ValueKind::MAP || values[i].map.length != keys.length

    keys.each_index do |j|
      return false if values[i].map[j].key != keys[j]
    end
  end
  true
end

.key_ref_field_identity(key, state) ⇒ Object 



1987
1988
1989
1990
1991
1992
 
# File 'lib/twilic/core/protocol.rb', line 1987

def key_ref_field_identity(key, state)
  s = key_ref_string(key, state)
  return nil if s == ""

  s
end

.key_ref_string(key, state) ⇒ Object 



1977
1978
1979
1980
1981
1982
1983
1984
1985
 
# File 'lib/twilic/core/protocol.rb', line 1977

def key_ref_string(key, state)
  if key.is_id
    s, ok = state.key_table.get_value(key.id)
    return s if ok

    return ""
  end
  key.literal
end

.lookup_map_field(value, key) ⇒ Object 



1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
 
# File 'lib/twilic/core/protocol.rb', line 1782

def lookup_map_field(value, key)
  return nil unless value.kind == Model::ValueKind::MAP

  value.map.each do |entry|
    if entry.key == key
      v = entry.value.clone_value
      return v
    end
  end
  nil
end

.new_twilic_codec ⇒ Object 



15
16
17
 
# File 'lib/twilic/core/protocol.rb', line 15

def self.new_twilic_codec
  TwilicCodec.new
end

.normalized_logical_type(raw) ⇒ Object 



1809
1810
1811
 
# File 'lib/twilic/core/protocol.rb', line 1809

def normalized_logical_type(raw)
  raw.strip.downcase
end

.rows_from_values(values) ⇒ Object 



1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
 
# File 'lib/twilic/core/protocol.rb', line 1813

def rows_from_values(values)
  rows = Array.new(values.length) { [] }
  values.each_with_index do |value, i|
    if value.kind == Model::ValueKind::ARRAY
      row = Array.new(value.arr.length)
      value.arr.each_with_index { |item, j| row[j] = item.clone_value }
      rows[i] = row
    else
      rows[i] = [value.clone_value]
    end
  end
  rows
end

.schema_present_field_indices(schema, presence, has_presence) ⇒ Object 



1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
 
# File 'lib/twilic/core/protocol.rb', line 1794

def schema_present_field_indices(schema, presence, has_presence)
  unless has_presence
    out = Array.new(schema.fields.length, 0)
    out.each_index { |i| out[i] = i }
    return out
  end
  raise Errors.invalid_data("presence bitmap mismatch for schema") if presence.length != schema.fields.length

  out = []
  schema.fields.each_with_index do |_field, i|
    out << i if presence[i]
  end
  out
end

.shape_values_to_map(keys, presence, has_presence, values) ⇒ Object 



1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
 
# File 'lib/twilic/core/protocol.rb', line 1994

def shape_values_to_map(keys, presence, has_presence, values)
  out = []
  idx = 0
  keys.each_with_index do |key, i|
    next if has_presence && i < presence.length && !presence[i]
    break if idx >= values.length

    out << Model.entry(key, values[idx].clone_value)
    idx += 1
  end
  out
end

.should_register_shape(keys, observed_count) ⇒ Object 



1923
1924
1925
 
# File 'lib/twilic/core/protocol.rb', line 1923

def should_register_shape(keys, observed_count)
  !keys.empty? && observed_count >= 2
end

.strip_nulls(values) ⇒ Object 



1846
1847
1848
1849
1850
1851
1852
 
# File 'lib/twilic/core/protocol.rb', line 1846

def strip_nulls(values)
  out = []
  values.each do |value|
    out << value unless value.kind == Model::ValueKind::NULL
  end
  out
end

.supports_state_patch(base, current) ⇒ Object 



1927
1928
1929
1930
1931
1932
1933
 
# File 'lib/twilic/core/protocol.rb', line 1927

def supports_state_patch(base, current)
  !base.nil? && !current.nil? && base.kind == current.kind &&
    (base.kind == Model::MessageKind::MAP ||
      base.kind == Model::MessageKind::SCHEMA_OBJECT ||
      base.kind == Model::MessageKind::SHAPED_OBJECT ||
      base.kind == Model::MessageKind::ARRAY)
end

.twilic_codec_with_options(options) ⇒ Object 



19
20
21
 
# File 'lib/twilic/core/protocol.rb', line 19

def self.twilic_codec_with_options(options)
  TwilicCodec.new(options)
end

.typed_vector_to_value(vector) ⇒ Object 



1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
 
# File 'lib/twilic/core/protocol.rb', line 1939

def typed_vector_to_value(vector)
  case vector.element_type
  when Model::ElementType::BOOL
    out = Array.new(vector.data.bools.length)
    out.each_index { |i| out[i] = Model.bool_value(vector.data.bools[i]) }
    Model.array_value(out)
  when Model::ElementType::I64
    out = Array.new(vector.data.i64s.length)
    out.each_index { |i| out[i] = Model.i64_value(vector.data.i64s[i]) }
    Model.array_value(out)
  when Model::ElementType::U64
    out = Array.new(vector.data.u64s.length)
    out.each_index { |i| out[i] = Model.u64_value(vector.data.u64s[i]) }
    Model.array_value(out)
  when Model::ElementType::F64
    out = Array.new(vector.data.f64s.length)
    out.each_index { |i| out[i] = Model.f64_value(vector.data.f64s[i]) }
    Model.array_value(out)
  when Model::ElementType::STRING
    out = Array.new(vector.data.strings.length)
    out.each_index { |i| out[i] = Model.string_value(vector.data.strings[i]) }
    Model.array_value(out)
  else
    Model.array_value([])
  end
end