Class: Aws::Glue::Types::S3JsonSource

Inherits:

Struct

• Object
• Struct
• Aws::Glue::Types::S3JsonSource

Includes:

Structure

Defined in:

lib/aws-sdk-glue/types.rb

Overview

Note:

When making an API call, you may pass S3JsonSource data as a hash:

{
  name: "NodeName", # required
  paths: ["EnclosedInStringProperty"], # required
  compression_type: "gzip", # accepts gzip, bzip2
  exclusions: ["EnclosedInStringProperty"],
  group_size: "EnclosedInStringProperty",
  group_files: "EnclosedInStringProperty",
  recurse: false,
  max_band: 1,
  max_files_in_band: 1,
  additional_options: {
    bounded_size: 1,
    bounded_files: 1,
    enable_sample_path: false,
    sample_path: "EnclosedInStringProperty",
  },
  json_path: "EnclosedInStringProperty",
  multiline: false,
  output_schemas: [
    {
      columns: [
        {
          name: "GlueStudioColumnNameString", # required
          type: "ColumnTypeString",
        },
      ],
    },
  ],
}

Specifies a JSON data store stored in Amazon S3.

See Also:

• AWS API Documentation

Constant Summary

SENSITIVE =

[]

Instance Attribute Summary

• #additional_options ⇒ Types::S3DirectSourceAdditionalOptions

  Specifies additional connection options.

• #compression_type ⇒ String

  Specifies how the data is compressed.

• #exclusions ⇒ Array<String>

  A string containing a JSON list of Unix-style glob patterns to exclude.

• #group_files ⇒ String

  Grouping files is turned on by default when the input contains more than 50,000 files.

• #group_size ⇒ String

  The target group size in bytes.

• #json_path ⇒ String

  A JsonPath string defining the JSON data.

• #max_band ⇒ Integer

  This option controls the duration in milliseconds after which the s3 listing is likely to be consistent.

• #max_files_in_band ⇒ Integer

  This option specifies the maximum number of files to save from the last maxBand seconds.

• #multiline ⇒ Boolean

  A Boolean value that specifies whether a single record can span multiple lines.

• #name ⇒ String

  The name of the data store.

• #output_schemas ⇒ Array<Types::GlueSchema>

  Specifies the data schema for the S3 JSON source.

• #paths ⇒ Array<String>

  A list of the Amazon S3 paths to read from.

• #recurse ⇒ Boolean

  If set to true, recursively reads files in all subdirectories under the specified paths.

Instance Attribute Details

#additional_options ⇒ Types::S3DirectSourceAdditionalOptions

Specifies additional connection options.

Returns:

• (Types::S3DirectSourceAdditionalOptions)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#compression_type ⇒ String

Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are `“gzip”` and `“bzip”`).

Returns:

• (String)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#exclusions ⇒ Array<String>

A string containing a JSON list of Unix-style glob patterns to exclude. For example, “[\”**.pdf\“]” excludes all PDF files.

Returns:

• (Array<String>)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#group_files ⇒ String

Grouping files is turned on by default when the input contains more than 50,000 files. To turn on grouping with fewer than 50,000 files, set this parameter to “inPartition”. To disable grouping when there are more than 50,000 files, set this parameter to `“none”`.

Returns:

• (String)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#group_size ⇒ String

The target group size in bytes. The default is computed based on the input data size and the size of your cluster. When there are fewer than 50,000 input files, `“groupFiles”` must be set to `“inPartition”` for this to take effect.

Returns:

• (String)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#json_path ⇒ String

A JsonPath string defining the JSON data.

Returns:

• (String)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#max_band ⇒ Integer

This option controls the duration in milliseconds after which the s3 listing is likely to be consistent. Files with modification timestamps falling within the last maxBand milliseconds are tracked specially when using JobBookmarks to account for Amazon S3 eventual consistency. Most users don't need to set this option. The default is 900000 milliseconds, or 15 minutes.

Returns:

• (Integer)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#max_files_in_band ⇒ Integer

This option specifies the maximum number of files to save from the last maxBand seconds. If this number is exceeded, extra files are skipped and only processed in the next job run.

Returns:

• (Integer)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#multiline ⇒ Boolean

A Boolean value that specifies whether a single record can span multiple lines. This can occur when a field contains a quoted new-line character. You must set this option to True if any record spans multiple lines. The default value is `False`, which allows for more aggressive file-splitting during parsing.

Returns:

• (Boolean)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#name ⇒ String

The name of the data store.

Returns:

• (String)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#output_schemas ⇒ Array<Types::GlueSchema>

Specifies the data schema for the S3 JSON source.

Returns:

• (Array<Types::GlueSchema>)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#paths ⇒ Array<String>

A list of the Amazon S3 paths to read from.

Returns:

• (Array<String>)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end

#recurse ⇒ Boolean

If set to true, recursively reads files in all subdirectories under the specified paths.

Returns:

• (Boolean)

# File 'lib/aws-sdk-glue/types.rb', line 20637

class S3JsonSource < Struct.new(
  :name,
  :paths,
  :compression_type,
  :exclusions,
  :group_size,
  :group_files,
  :recurse,
  :max_band,
  :max_files_in_band,
  :additional_options,
  :json_path,
  :multiline,
  :output_schemas)
  SENSITIVE = []
  include Aws::Structure
end