Updated bigquery module (#1861)
This commit is contained in:
luigi-bitonti
GitHub
parent
6112d06706
commit
b5cd2d8088
|
|
@ -294,10 +294,10 @@ module "bigquery-dataset" {
|
|||
| [iam](variables.tf#L92) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [labels](variables.tf#L103) | Dataset labels. | <code>map(string)</code> | | <code>{}</code> |
|
||||
| [location](variables.tf#L109) | Dataset location. | <code>string</code> | | <code>"EU"</code> |
|
||||
| [materialized_views](variables.tf#L115) | Materialized views definitions. | <code title="map(object({ query = string allow_non_incremental_definition = optional(bool) deletion_protection = optional(bool) description = optional(string, "Terraform managed.") enable_refresh = optional(bool) friendly_name = optional(string) labels = optional(map(string), {}) refresh_interval_ms = optional(bool) options = optional(object({ clustering = optional(list(string)) expiration_time = optional(number) }), {}) partitioning = optional(object({ field = optional(string) range = optional(object({ end = number interval = number start = number })) time = optional(object({ type = string expiration_ms = optional(number) field = optional(string) require_partition_filter = optional(bool) })) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [materialized_views](variables.tf#L115) | Materialized views definitions. | <code title="map(object({ query = string allow_non_incremental_definition = optional(bool) deletion_protection = optional(bool) description = optional(string, "Terraform managed.") enable_refresh = optional(bool) friendly_name = optional(string) labels = optional(map(string), {}) refresh_interval_ms = optional(bool) require_partition_filter = optional(bool) options = optional(object({ clustering = optional(list(string)) expiration_time = optional(number) }), {}) partitioning = optional(object({ field = optional(string) range = optional(object({ end = number interval = number start = number })) time = optional(object({ type = string expiration_ms = optional(number) field = optional(string) })) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [options](variables.tf#L148) | Dataset options. | <code title="object({ default_collation = optional(string) default_table_expiration_ms = optional(number) default_partition_expiration_ms = optional(number) delete_contents_on_destroy = optional(bool, false) is_case_insensitive = optional(bool) max_time_travel_hours = optional(number, 168) storage_billing_model = optional(string) })">object({…})</code> | | <code>{}</code> |
|
||||
| [tables](variables.tf#L167) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | <code title="map(object({ deletion_protection = optional(bool) description = optional(string, "Terraform managed.") friendly_name = optional(string) labels = optional(map(string), {}) schema = optional(string) options = optional(object({ clustering = optional(list(string)) encryption_key = optional(string) expiration_time = optional(number) }), {}) partitioning = optional(object({ field = optional(string) range = optional(object({ end = number interval = number start = number })) time = optional(object({ type = string expiration_ms = optional(number) field = optional(string) require_partition_filter = optional(bool) })) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [views](variables.tf#L198) | View definitions. | <code title="map(object({ query = string deletion_protection = optional(bool) description = optional(string, "Terraform managed.") friendly_name = optional(string) labels = optional(map(string), {}) use_legacy_sql = optional(bool) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [tables](variables.tf#L167) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | <code title="map(object({ deletion_protection = optional(bool) description = optional(string, "Terraform managed.") friendly_name = optional(string) labels = optional(map(string), {}) require_partition_filter = optional(bool) schema = optional(string) external_data_configuration = optional(object({ autodetect = bool source_uris = list(string) avro_logical_types = optional(bool) compression = optional(string) connection_id = optional(string) file_set_spec_type = optional(string) ignore_unknown_values = optional(bool) metadata_cache_mode = optional(string) object_metadata = optional(string) json_options_encoding = optional(string) reference_file_schema_uri = optional(string) schema = optional(string) source_format = optional(string) max_bad_records = optional(number) csv_options = optional(object({ quote = string allow_jagged_rows = optional(bool) allow_quoted_newlines = optional(bool) encoding = optional(string) field_delimiter = optional(string) skip_leading_rows = optional(number) })) google_sheets_options = optional(object({ range = optional(string) skip_leading_rows = optional(number) })) hive_partitioning_options = optional(object({ mode = optional(string) require_partition_filter = optional(bool) source_uri_prefix = optional(string) })) parquet_options = optional(object({ enum_as_string = optional(bool) enable_list_inference = optional(bool) })) })) options = optional(object({ clustering = optional(list(string)) encryption_key = optional(string) expiration_time = optional(number) max_staleness = optional(string) }), {}) partitioning = optional(object({ field = optional(string) range = optional(object({ end = number interval = number start = number })) time = optional(object({ type = string expiration_ms = optional(number) field = optional(string) })) })) table_constraints = optional(object({ primary_key_columns = optional(list(string)) foreign_keys = optional(object({ referenced_table = object({ project_id = string dataset_id = string table_id = string }) column_references = object({ referencing_column = string referenced_column = string }) name = optional(string) })) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [views](variables.tf#L252) | View definitions. | <code title="map(object({ query = string deletion_protection = optional(bool) description = optional(string, "Terraform managed.") friendly_name = optional(string) labels = optional(map(string), {}) use_legacy_sql = optional(bool) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -209,18 +209,20 @@ resource "google_bigquery_dataset_iam_binding" "bindings" {
|
|||
}
|
||||
|
||||
resource "google_bigquery_table" "default" {
|
||||
provider = google-beta
|
||||
for_each = var.tables
|
||||
project = var.project_id
|
||||
dataset_id = google_bigquery_dataset.default.dataset_id
|
||||
table_id = each.key
|
||||
friendly_name = each.value.friendly_name
|
||||
description = each.value.description
|
||||
clustering = each.value.options.clustering
|
||||
expiration_time = each.value.options.expiration_time
|
||||
labels = each.value.labels
|
||||
schema = each.value.schema
|
||||
deletion_protection = each.value.deletion_protection
|
||||
provider = google-beta
|
||||
for_each = var.tables
|
||||
project = var.project_id
|
||||
dataset_id = google_bigquery_dataset.default.dataset_id
|
||||
table_id = each.key
|
||||
friendly_name = each.value.friendly_name
|
||||
description = each.value.description
|
||||
clustering = each.value.options.clustering
|
||||
expiration_time = each.value.options.expiration_time
|
||||
labels = each.value.labels
|
||||
max_staleness = each.value.options.max_staleness
|
||||
schema = each.value.schema
|
||||
deletion_protection = each.value.deletion_protection
|
||||
require_partition_filter = each.value.require_partition_filter
|
||||
|
||||
dynamic "encryption_configuration" {
|
||||
for_each = each.value.options.encryption_key != null ? [""] : []
|
||||
|
|
@ -229,6 +231,97 @@ resource "google_bigquery_table" "default" {
|
|||
}
|
||||
}
|
||||
|
||||
dynamic "external_data_configuration" {
|
||||
for_each = each.value.external_data_configuration != null ? [""] : []
|
||||
content {
|
||||
autodetect = each.value.external_data_configuration.autodetect
|
||||
compression = each.value.external_data_configuration.compression
|
||||
connection_id = each.value.external_data_configuration.connection_id
|
||||
file_set_spec_type = each.value.external_data_configuration.file_set_spec_type
|
||||
ignore_unknown_values = each.value.external_data_configuration.ignore_unknown_values
|
||||
max_bad_records = each.value.external_data_configuration.max_bad_records
|
||||
metadata_cache_mode = each.value.external_data_configuration.metadata_cache_mode
|
||||
object_metadata = each.value.external_data_configuration.object_metadata
|
||||
reference_file_schema_uri = each.value.external_data_configuration.reference_file_schema_uri
|
||||
schema = each.value.external_data_configuration.schema
|
||||
source_format = each.value.external_data_configuration.source_format
|
||||
source_uris = each.value.external_data_configuration.source_uris
|
||||
|
||||
dynamic "avro_options" {
|
||||
for_each = each.value.external_data_configuration.avro_logical_types != null ? [""] : []
|
||||
content {
|
||||
use_avro_logical_types = each.value.external_data_configuration.avro_logical_types
|
||||
}
|
||||
}
|
||||
dynamic "csv_options" {
|
||||
for_each = each.value.external_data_configuration.csv_options != null ? [""] : []
|
||||
content {
|
||||
quote = each.value.external_data_configuration.csv_options.quote
|
||||
allow_jagged_rows = each.value.external_data_configuration.csv_options.allow_jagged_rows
|
||||
allow_quoted_newlines = each.value.external_data_configuration.csv_options.allow_quoted_newlines
|
||||
encoding = each.value.external_data_configuration.csv_options.encoding
|
||||
field_delimiter = each.value.external_data_configuration.csv_options.field_delimiter
|
||||
skip_leading_rows = each.value.external_data_configuration.csv_options.skip_leading_rows
|
||||
}
|
||||
}
|
||||
dynamic "json_options" {
|
||||
for_each = each.value.external_data_configuration.json_options_encoding != null ? [""] : []
|
||||
content {
|
||||
encoding = each.value.external_data_configuration.json_options_encoding
|
||||
}
|
||||
}
|
||||
dynamic "google_sheets_options" {
|
||||
for_each = each.value.external_data_configuration.google_sheets_options != null ? [""] : []
|
||||
content {
|
||||
range = each.value.external_data_configuration.google_sheets_options.range
|
||||
skip_leading_rows = each.value.external_data_configuration.google_sheets_options.skip_leading_rows
|
||||
}
|
||||
}
|
||||
dynamic "hive_partitioning_options" {
|
||||
for_each = each.value.external_data_configuration.hive_partitioning_options != null ? [""] : []
|
||||
content {
|
||||
mode = each.value.external_data_configuration.hive_partitioning_options.mode
|
||||
require_partition_filter = each.value.external_data_configuration.hive_partitioning_options.require_partition_filter
|
||||
source_uri_prefix = each.value.external_data_configuration.hive_partitioning_options.source_uri_prefix
|
||||
}
|
||||
}
|
||||
dynamic "parquet_options" {
|
||||
for_each = each.value.external_data_configuration.parquet_options != null ? [""] : []
|
||||
content {
|
||||
enum_as_string = each.value.external_data_configuration.parquet_options.enum_as_string
|
||||
enable_list_inference = each.value.external_data_configuration.parquet_options.enable_list_inference
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dynamic "table_constraints" {
|
||||
for_each = each.value.table_constraints != null ? [""] : []
|
||||
content {
|
||||
dynamic "primary_key" {
|
||||
for_each = each.value.table_constraints.primary_key_columns != null ? [""] : []
|
||||
content {
|
||||
columns = each.value.table_constraints.primary_key_columns
|
||||
}
|
||||
}
|
||||
dynamic "foreign_keys" {
|
||||
for_each = each.value.table_constraints.foreign_keys != null ? [""] : []
|
||||
content {
|
||||
name = each.value.table_constraints.foreign_keys.name
|
||||
referenced_table {
|
||||
project_id = each.value.table_constraints.foreign_keys.referenced_table.project_id
|
||||
dataset_id = each.value.table_constraints.foreign_keys.referenced_table.dataset_id
|
||||
table_id = each.value.table_constraints.foreign_keys.referenced_table.table_id
|
||||
}
|
||||
column_references {
|
||||
referencing_column = each.value.table_constraints.foreign_keys.column_references.referencing_column
|
||||
referenced_column = each.value.table_constraints.foreign_keys.column_references.referenced_column
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dynamic "range_partitioning" {
|
||||
for_each = try(each.value.partitioning.range, null) != null ? [""] : []
|
||||
content {
|
||||
|
|
@ -244,10 +337,9 @@ resource "google_bigquery_table" "default" {
|
|||
dynamic "time_partitioning" {
|
||||
for_each = try(each.value.partitioning.time, null) != null ? [""] : []
|
||||
content {
|
||||
expiration_ms = each.value.partitioning.time.expiration_ms
|
||||
field = each.value.partitioning.time.field
|
||||
type = each.value.partitioning.time.type
|
||||
require_partition_filter = each.value.partitioning.time.require_partition_filter
|
||||
expiration_ms = each.value.partitioning.time.expiration_ms
|
||||
field = each.value.partitioning.time.field
|
||||
type = each.value.partitioning.time.type
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -270,17 +362,18 @@ resource "google_bigquery_table" "views" {
|
|||
}
|
||||
|
||||
resource "google_bigquery_table" "materialized_view" {
|
||||
depends_on = [google_bigquery_table.default]
|
||||
for_each = var.materialized_views
|
||||
project = var.project_id
|
||||
dataset_id = google_bigquery_dataset.default.dataset_id
|
||||
table_id = each.key
|
||||
friendly_name = each.value.friendly_name
|
||||
description = each.value.description
|
||||
labels = each.value.labels
|
||||
clustering = each.value.options.clustering
|
||||
expiration_time = each.value.options.expiration_time
|
||||
deletion_protection = each.value.deletion_protection
|
||||
depends_on = [google_bigquery_table.default]
|
||||
for_each = var.materialized_views
|
||||
project = var.project_id
|
||||
dataset_id = google_bigquery_dataset.default.dataset_id
|
||||
table_id = each.key
|
||||
friendly_name = each.value.friendly_name
|
||||
description = each.value.description
|
||||
labels = each.value.labels
|
||||
clustering = each.value.options.clustering
|
||||
expiration_time = each.value.options.expiration_time
|
||||
deletion_protection = each.value.deletion_protection
|
||||
require_partition_filter = each.value.require_partition_filter
|
||||
|
||||
dynamic "range_partitioning" {
|
||||
for_each = try(each.value.partitioning.range, null) != null ? [""] : []
|
||||
|
|
@ -297,10 +390,9 @@ resource "google_bigquery_table" "materialized_view" {
|
|||
dynamic "time_partitioning" {
|
||||
for_each = try(each.value.partitioning.time, null) != null ? [""] : []
|
||||
content {
|
||||
expiration_ms = each.value.partitioning.time.expiration_ms
|
||||
field = each.value.partitioning.time.field
|
||||
type = each.value.partitioning.time.type
|
||||
require_partition_filter = each.value.partitioning.time.require_partition_filter
|
||||
expiration_ms = each.value.partitioning.time.expiration_ms
|
||||
field = each.value.partitioning.time.field
|
||||
type = each.value.partitioning.time.type
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -123,6 +123,7 @@ variable "materialized_views" {
|
|||
friendly_name = optional(string)
|
||||
labels = optional(map(string), {})
|
||||
refresh_interval_ms = optional(bool)
|
||||
require_partition_filter = optional(bool)
|
||||
options = optional(object({
|
||||
clustering = optional(list(string))
|
||||
expiration_time = optional(number)
|
||||
|
|
@ -135,10 +136,9 @@ variable "materialized_views" {
|
|||
start = number
|
||||
}))
|
||||
time = optional(object({
|
||||
type = string
|
||||
expiration_ms = optional(number)
|
||||
field = optional(string)
|
||||
require_partition_filter = optional(bool)
|
||||
type = string
|
||||
expiration_ms = optional(number)
|
||||
field = optional(string)
|
||||
}))
|
||||
}))
|
||||
}))
|
||||
|
|
@ -167,15 +167,55 @@ variable "project_id" {
|
|||
variable "tables" {
|
||||
description = "Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null."
|
||||
type = map(object({
|
||||
deletion_protection = optional(bool)
|
||||
description = optional(string, "Terraform managed.")
|
||||
friendly_name = optional(string)
|
||||
labels = optional(map(string), {})
|
||||
schema = optional(string)
|
||||
deletion_protection = optional(bool)
|
||||
description = optional(string, "Terraform managed.")
|
||||
friendly_name = optional(string)
|
||||
labels = optional(map(string), {})
|
||||
require_partition_filter = optional(bool)
|
||||
schema = optional(string)
|
||||
external_data_configuration = optional(object({
|
||||
autodetect = bool
|
||||
source_uris = list(string)
|
||||
avro_logical_types = optional(bool)
|
||||
compression = optional(string)
|
||||
connection_id = optional(string)
|
||||
file_set_spec_type = optional(string)
|
||||
ignore_unknown_values = optional(bool)
|
||||
metadata_cache_mode = optional(string)
|
||||
object_metadata = optional(string)
|
||||
json_options_encoding = optional(string)
|
||||
reference_file_schema_uri = optional(string)
|
||||
schema = optional(string)
|
||||
source_format = optional(string)
|
||||
max_bad_records = optional(number)
|
||||
csv_options = optional(object({
|
||||
quote = string
|
||||
allow_jagged_rows = optional(bool)
|
||||
allow_quoted_newlines = optional(bool)
|
||||
encoding = optional(string)
|
||||
field_delimiter = optional(string)
|
||||
skip_leading_rows = optional(number)
|
||||
}))
|
||||
google_sheets_options = optional(object({
|
||||
range = optional(string)
|
||||
skip_leading_rows = optional(number)
|
||||
}))
|
||||
hive_partitioning_options = optional(object({
|
||||
mode = optional(string)
|
||||
require_partition_filter = optional(bool)
|
||||
source_uri_prefix = optional(string)
|
||||
}))
|
||||
parquet_options = optional(object({
|
||||
enum_as_string = optional(bool)
|
||||
enable_list_inference = optional(bool)
|
||||
}))
|
||||
|
||||
}))
|
||||
options = optional(object({
|
||||
clustering = optional(list(string))
|
||||
encryption_key = optional(string)
|
||||
expiration_time = optional(number)
|
||||
max_staleness = optional(string)
|
||||
}), {})
|
||||
partitioning = optional(object({
|
||||
field = optional(string)
|
||||
|
|
@ -185,10 +225,24 @@ variable "tables" {
|
|||
start = number
|
||||
}))
|
||||
time = optional(object({
|
||||
type = string
|
||||
expiration_ms = optional(number)
|
||||
field = optional(string)
|
||||
require_partition_filter = optional(bool)
|
||||
type = string
|
||||
expiration_ms = optional(number)
|
||||
field = optional(string)
|
||||
}))
|
||||
}))
|
||||
table_constraints = optional(object({
|
||||
primary_key_columns = optional(list(string))
|
||||
foreign_keys = optional(object({
|
||||
referenced_table = object({
|
||||
project_id = string
|
||||
dataset_id = string
|
||||
table_id = string
|
||||
})
|
||||
column_references = object({
|
||||
referencing_column = string
|
||||
referenced_column = string
|
||||
})
|
||||
name = optional(string)
|
||||
}))
|
||||
}))
|
||||
}))
|
||||
|
|
|
|||
Loading…
Reference in New Issue