Class: Multiwoven::Integrations::Source::Bigquery::Client

Inherits:
SourceConnector
  • Object
show all
Defined in:
lib/multiwoven/integrations/source/bigquery/client.rb

Instance Method Summary collapse

Instance Method Details

#check_connection(connection_config) ⇒ Object 



9
10
11
12
13
14
15
16
 
# File 'lib/multiwoven/integrations/source/bigquery/client.rb', line 9

def check_connection(connection_config)
  connection_config = connection_config.with_indifferent_access
  bigquery = create_connection(connection_config)
  bigquery.datasets
  ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
rescue StandardError => e
  ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
end

#create_connection(connection_config) ⇒ Object 



69
70
71
72
73
74
 
# File 'lib/multiwoven/integrations/source/bigquery/client.rb', line 69

def create_connection(connection_config)
  Google::Cloud::Bigquery.new(
    project: connection_config["project_id"],
    credentials: connection_config["credentials_json"]
  )
end

#create_streams(records) ⇒ Object 



76
77
78
79
80
 
# File 'lib/multiwoven/integrations/source/bigquery/client.rb', line 76

def create_streams(records)
  group_by_table(records).map do |r|
    Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
  end
end

#discover(connection_config) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/multiwoven/integrations/source/bigquery/client.rb', line 18

def discover(connection_config)
  connection_config = connection_config.with_indifferent_access
  bigquery = create_connection(connection_config)
  target_dataset_id = connection_config["dataset_id"]
  records = bigquery.datasets.flat_map do |dataset|
    next unless dataset.dataset_id == target_dataset_id

    dataset.tables.flat_map do |table|
      table.schema.fields.map do |field|
        {
          table_name: table.table_id,
          column_name: field.name,
          data_type: field.type,
          is_nullable: field.mode == "NULLABLE"
        }
      end
    end
  end
  catalog = Catalog.new(streams: create_streams(records))
  catalog.to_multiwoven_message
rescue StandardError => e
  handle_exception(
    "BIGQUERY:DISCOVER:EXCEPTION",
    "error",
    e
  )
end

#group_by_table(records) ⇒ Object 



82
83
84
85
86
87
88
89
 
# File 'lib/multiwoven/integrations/source/bigquery/client.rb', line 82

def group_by_table(records)
  records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
    {
      tablename: table_name,
      columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
    }
  end
end

#read(sync_config) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/multiwoven/integrations/source/bigquery/client.rb', line 46

def read(sync_config)
  connection_config = sync_config.source.connection_specification
  connection_config = connection_config.with_indifferent_access
  query = sync_config.model.query

  query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?

  bigquery = create_connection(connection_config)
  records = []
  results = bigquery.query(query) || []
  results.each do |row|
    records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
  end

  records
rescue StandardError => e
  handle_exception(
    "BIGQUERY:READ:EXCEPTION",
    "error",
    e
  )
end