From f2fc67d72d67d0fe91d213d5dd04acd420a59d63 Mon Sep 17 00:00:00 2001 From: Luigi Bitonti Date: Wed, 13 Sep 2023 15:13:13 +0200 Subject: [PATCH] Added possibility to use gcs push endpoint on pubsub subscription --- modules/pubsub/README.md | 56 +++++++++++++++++++++++++++---------- modules/pubsub/main.tf | 22 +++++++++++++++ modules/pubsub/variables.tf | 15 ++++++++++ 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 44a0e737..d7351ac8 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -124,6 +124,33 @@ module "pubsub" { # tftest modules=1 resources=2 ``` +### Cloud Storage subscriptions + +Cloud Storage subscriptions need extra configuration in the `cloud_storage_subscription_configs` variable. + +```hcl +module "pubsub" { + source = "./fabric/modules/pubsub" + pproject_id = "my-project" + name = "my-topic" + subscriptions = { + test-cloudstorage = null + } + cloud_storage_subscription_configs = { + test-cloudstorage = { + bucket = "my-bucket" + filename_prefix = "test_prefix" + filename_suffix = "test_suffix" + max_duration = "100s" + max_bytes = 1000 + avro_config = { + write_metadata = true + } + } + } +} +# tftest modules=1 resources=2 +``` ### Subscriptions with IAM ```hcl @@ -144,25 +171,25 @@ module "pubsub" { # tftest modules=1 resources=3 ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L79) | PubSub topic name. | string | ✓ | | -| [project_id](variables.tf#L84) | Project used for resources. | string | ✓ | | +| [name](variables.tf#L94) | PubSub topic name. | string | ✓ | | +| [project_id](variables.tf#L99) | Project used for resources. | string | ✓ | | | [bigquery_subscription_configs](variables.tf#L17) | Configuration parameters for BigQuery subscriptions. | map(object({…})) | | {} | -| [dead_letter_configs](variables.tf#L28) | Per-subscription dead letter policy configuration. | map(object({…})) | | {} | -| [defaults](variables.tf#L37) | Subscription defaults for options. | object({…}) | | {…} | -| [iam](variables.tf#L55) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [kms_key](variables.tf#L61) | KMS customer managed encryption key. | string | | null | -| [labels](variables.tf#L67) | Labels. | map(string) | | {} | -| [message_retention_duration](variables.tf#L73) | Minimum duration to retain a message after it is published to the topic. | string | | null | -| [push_configs](variables.tf#L89) | Push subscription configurations. | map(object({…})) | | {} | -| [regions](variables.tf#L102) | List of regions used to set persistence policy. | list(string) | | [] | -| [schema](variables.tf#L108) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | -| [subscription_iam](variables.tf#L118) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [subscriptions](variables.tf#L124) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | +| [cloud_storage_subscription_configs](variables.tf#L28) | Configuration parameters for Cloud Storage subscriptions. | map(object({…})) | | {} | +| [dead_letter_configs](variables.tf#L43) | Per-subscription dead letter policy configuration. | map(object({…})) | | {} | +| [defaults](variables.tf#L52) | Subscription defaults for options. | object({…}) | | {…} | +| [iam](variables.tf#L70) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [kms_key](variables.tf#L76) | KMS customer managed encryption key. | string | | null | +| [labels](variables.tf#L82) | Labels. | map(string) | | {} | +| [message_retention_duration](variables.tf#L88) | Minimum duration to retain a message after it is published to the topic. | string | | null | +| [push_configs](variables.tf#L104) | Push subscription configurations. | map(object({…})) | | {} | +| [regions](variables.tf#L117) | List of regions used to set persistence policy. | list(string) | | [] | +| [schema](variables.tf#L123) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | +| [subscription_iam](variables.tf#L133) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [subscriptions](variables.tf#L139) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | ## Outputs @@ -174,5 +201,4 @@ module "pubsub" { | [subscription_id](outputs.tf#L36) | Subscription ids. | | | [subscriptions](outputs.tf#L46) | Subscription resources. | | | [topic](outputs.tf#L54) | Topic resource. | | - diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index ccb6f5d7..2a7f873f 100644 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -24,6 +24,9 @@ locals { } ] ]) + metadata_config = { + for k, v in var.cloud_storage_subscription_configs : k => v.avro_config + } oidc_config = { for k, v in var.push_configs : k => v.oidc_token } @@ -127,6 +130,25 @@ resource "google_pubsub_subscription" "default" { drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields } } + + dynamic "cloud_storage_config" { + for_each = try(var.cloud_storage_subscription_configs[each.key], null) == null ? [] : [""] + content { + bucket = var.cloud_storage_subscription_configs[each.key].bucket + filename_prefix = var.cloud_storage_subscription_configs[each.key].filename_prefix + filename_suffix = var.cloud_storage_subscription_configs[each.key].filename_suffix + max_duration = var.cloud_storage_subscription_configs[each.key].max_duration + max_bytes = var.cloud_storage_subscription_configs[each.key].max_bytes + dynamic "avro_config" { + for_each = ( + local.metadata_config[each.key] == null ? [] : [""] + ) + content { + write_metadata = local.metadata_config[each.key].write_metadata + } + } + } + } } resource "google_pubsub_subscription_iam_binding" "default" { diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index afefb4a8..35dfc172 100644 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -25,6 +25,21 @@ variable "bigquery_subscription_configs" { default = {} } +variable "cloud_storage_subscription_configs" { + description = "Configuration parameters for Cloud Storage subscriptions." + type = map(object({ + bucket = string + filename_prefix = string + filename_suffix = string + max_duration = string + max_bytes = number + avro_config = object({ + write_metadata = bool + }) + })) + default = {} +} + variable "dead_letter_configs" { description = "Per-subscription dead letter policy configuration." type = map(object({