From e692eac8677cf19828b064e2a4a082a910ec298f Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Mon, 21 Nov 2022 17:43:17 +0100 Subject: [PATCH] Add BigQuery subcriptions to Pubsub module. Pubsub can now have subscriptions that write directly to BigQuery. * https://cloud.google.com/pubsub/docs/bigquery In the Google Terraform provider, this is configured using an additional block inside a `google_pubsub_subscription` resource. * https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription#nested_bigquery_config This PR adds a new input variable to the `pubsub` module, to optionally add this block to some of the subscriptions defined in the module. --- modules/pubsub/README.md | 52 +++++++++++++++++++++++++++---------- modules/pubsub/main.tf | 10 +++++++ modules/pubsub/variables.tf | 11 ++++++++ 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 61e4aaf5..b204ff56 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -2,7 +2,6 @@ This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels, as well as schemas. - ## Examples ### Simple topic with IAM @@ -101,6 +100,30 @@ module "pubsub" { # tftest modules=1 resources=2 ``` +### BigQuery subscriptions + +BigQuery subscriptions need extra configuration in the `bigquery_subscription_configs` variable. + +```hcl +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + subscriptions = { + test-bigquery = null + } + bigquery_subscription_configs = { + test-bigquery = { + table = "my_project_id:my_dataset.my_table" + use_topic_schema = true + write_metadata = false + drop_unknown_fields = true + } + } +} +# tftest modules=1 resources=2 +``` + ### Subscriptions with IAM ```hcl @@ -126,19 +149,20 @@ module "pubsub" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L68) | PubSub topic name. | string | ✓ | | -| [project_id](variables.tf#L73) | Project used for resources. | string | ✓ | | -| [dead_letter_configs](variables.tf#L17) | Per-subscription dead letter policy configuration. | map(object({…})) | | {} | -| [defaults](variables.tf#L26) | Subscription defaults for options. | object({…}) | | {…} | -| [iam](variables.tf#L44) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [kms_key](variables.tf#L50) | KMS customer managed encryption key. | string | | null | -| [labels](variables.tf#L56) | Labels. | map(string) | | {} | -| [message_retention_duration](variables.tf#L62) | Minimum duration to retain a message after it is published to the topic. | string | | null | -| [push_configs](variables.tf#L78) | Push subscription configurations. | map(object({…})) | | {} | -| [regions](variables.tf#L91) | List of regions used to set persistence policy. | list(string) | | [] | -| [schema](variables.tf#L97) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | -| [subscription_iam](variables.tf#L107) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [subscriptions](variables.tf#L113) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | +| [name](variables.tf#L79) | PubSub topic name. | string | ✓ | | +| [project_id](variables.tf#L84) | Project used for resources. | string | ✓ | | +| [bigquery_subscription_configs](variables.tf#L17) | Configuration parameters for BigQuery subscriptions. | map(object({…})) | | {} | +| [dead_letter_configs](variables.tf#L28) | Per-subscription dead letter policy configuration. | map(object({…})) | | {} | +| [defaults](variables.tf#L37) | Subscription defaults for options. | object({…}) | | {…} | +| [iam](variables.tf#L55) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [kms_key](variables.tf#L61) | KMS customer managed encryption key. | string | | null | +| [labels](variables.tf#L67) | Labels. | map(string) | | {} | +| [message_retention_duration](variables.tf#L73) | Minimum duration to retain a message after it is published to the topic. | string | | null | +| [push_configs](variables.tf#L89) | Push subscription configurations. | map(object({…})) | | {} | +| [regions](variables.tf#L102) | List of regions used to set persistence policy. | list(string) | | [] | +| [schema](variables.tf#L108) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | +| [subscription_iam](variables.tf#L118) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [subscriptions](variables.tf#L124) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | ## Outputs diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index aa63e8c0..37717bf3 100644 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -116,6 +116,16 @@ resource "google_pubsub_subscription" "default" { } } } + + dynamic "bigquery_config" { + for_each = try(var.bigquery_subscription_configs[each.key], null) == null ? [] : [""] + content { + table = var.bigquery_subscription_configs[each.key].table + use_topic_schema = var.bigquery_subscription_configs[each.key].use_topic_schema + write_metadata = var.bigquery_subscription_configs[each.key].write_metadata + drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields + } + } } resource "google_pubsub_subscription_iam_binding" "default" { diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index c84168ea..afefb4a8 100644 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -14,6 +14,17 @@ * limitations under the License. */ +variable "bigquery_subscription_configs" { + description = "Configuration parameters for BigQuery subscriptions." + type = map(object({ + table = string + use_topic_schema = bool + write_metadata = bool + drop_unknown_fields = bool + })) + default = {} +} + variable "dead_letter_configs" { description = "Per-subscription dead letter policy configuration." type = map(object({