Class: Multiwoven::Integrations::Source::AmazonS3::Client

Inherits:
SourceConnector
  • Object
show all
Defined in:
lib/multiwoven/integrations/source/amazon_s3/client.rb

Constant Summary collapse

DISCOVER_QUERY = 
"SELECT * FROM S3Object LIMIT 1;"

Instance Method Summary collapse

Instance Method Details

#check_connection(connection_config) ⇒ Object 



9
10
11
12
13
14
15
16
17
 
# File 'lib/multiwoven/integrations/source/amazon_s3/client.rb', line 9

def check_connection(connection_config)
  connection_config = connection_config.with_indifferent_access
  auth_data = get_auth_data(connection_config)
  client = config_aws(auth_data, connection_config[:region])
  client.get_bucket_location({ bucket: connection_config[:bucket] })
  ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
rescue StandardError => e
  ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
end

#discover(connection_config) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/multiwoven/integrations/source/amazon_s3/client.rb', line 19

def discover(connection_config)
  connection_config = connection_config.with_indifferent_access
  auth_data = get_auth_data(connection_config)
  connection_config[:access_id] = auth_data.credentials.access_key_id
  connection_config[:secret_access] = auth_data.credentials.secret_access_key
  connection_config[:session_token] = auth_data.credentials.session_token
  conn = create_connection(connection_config)
  # If pulling from multiple files, all files must have the same schema
  path = build_path(connection_config[:path])
  full_path = "s3://#{connection_config[:bucket]}/#{path}*.#{connection_config[:file_type]}"
  records = get_results(conn, "DESCRIBE SELECT * FROM '#{full_path}';")
  columns = build_discover_columns(records)
  streams = [Multiwoven::Integrations::Protocol::Stream.new(name: full_path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
  catalog = Catalog.new(streams: streams)
  catalog.to_multiwoven_message
rescue StandardError => e
  handle_exception(e, { context: "AMAZONS3:DISCOVER:EXCEPTION", type: "error" })
end

#read(sync_config) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/multiwoven/integrations/source/amazon_s3/client.rb', line 38

def read(sync_config)
  connection_config = sync_config.source.connection_specification.with_indifferent_access
  auth_data = get_auth_data(connection_config)
  connection_config[:access_id] = auth_data.credentials.access_key_id
  connection_config[:secret_access] = auth_data.credentials.secret_access_key
  connection_config[:session_token] = auth_data.credentials.session_token
  conn = create_connection(connection_config)
  query = sync_config.model.query
  query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
  query(conn, query)
rescue StandardError => e
  handle_exception(e, {
                     context: "AMAZONS3:READ:EXCEPTION",
                     type: "error",
                     sync_id: sync_config.sync_id,
                     sync_run_id: sync_config.sync_run_id
                   })
end