Class: Purplelight::WriterParquet

Inherits:
Object
  • Object
show all
Defined in:
lib/purplelight/writer_parquet.rb

Overview

WriterParquet writes Parquet files via Apache Arrow when available.

Constant Summary collapse

DEFAULT_ROW_GROUP_SIZE = 
10_000
DICTIONARY_CARDINALITY_LIMIT = 
16
DICTIONARY_MIN_REPETITIONS = 
4

Instance Method Summary collapse

Constructor Details

#initialize(directory:, prefix:, compression: :zstd, row_group_size: DEFAULT_ROW_GROUP_SIZE, logger: nil, manifest: nil, single_file: true, schema: nil, rotate_rows: nil) ⇒ WriterParquet

Returns a new instance of WriterParquet.



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/purplelight/writer_parquet.rb', line 19

def initialize(directory:, prefix:, compression: :zstd, row_group_size: DEFAULT_ROW_GROUP_SIZE, logger: nil,
               manifest: nil, single_file: true, schema: nil, rotate_rows: nil)
  @directory = directory
  @prefix = prefix
  @compression = compression
  @row_group_size = row_group_size
  @logger = logger
  @manifest = manifest
  @single_file = single_file
  @schema = schema
  @rotate_rows = rotate_rows

  @closed = false
  @file_seq = manifest ? manifest.parts.length : 0
  @part_index = nil
  @pq_writer = nil
  @rows_in_current_file = 0

  ensure_dependencies!
  reset_buffers
end

Instance Method Details

#close ⇒ Object 



46
47
48
49
50
51
52
 
# File 'lib/purplelight/writer_parquet.rb', line 46

def close
  return if @closed

  flush_all_row_groups
  finalize_current_part! if @writer_path
  @closed = true
end

#write_many(array_of_docs) ⇒ Object 



41
42
43
44
 
# File 'lib/purplelight/writer_parquet.rb', line 41

def write_many(array_of_docs)
  @buffer_docs.concat(array_of_docs)
  flush_row_groups_if_needed
end