diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md
index 44a0e737..d7351ac8 100644
--- a/modules/pubsub/README.md
+++ b/modules/pubsub/README.md
@@ -124,6 +124,33 @@ module "pubsub" {
# tftest modules=1 resources=2
```
+### Cloud Storage subscriptions
+
+Cloud Storage subscriptions need extra configuration in the `cloud_storage_subscription_configs` variable.
+
+```hcl
+module "pubsub" {
+ source = "./fabric/modules/pubsub"
+ pproject_id = "my-project"
+ name = "my-topic"
+ subscriptions = {
+ test-cloudstorage = null
+ }
+ cloud_storage_subscription_configs = {
+ test-cloudstorage = {
+ bucket = "my-bucket"
+ filename_prefix = "test_prefix"
+ filename_suffix = "test_suffix"
+ max_duration = "100s"
+ max_bytes = 1000
+ avro_config = {
+ write_metadata = true
+ }
+ }
+ }
+}
+# tftest modules=1 resources=2
+```
### Subscriptions with IAM
```hcl
@@ -144,25 +171,25 @@ module "pubsub" {
# tftest modules=1 resources=3
```
-
## Variables
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [name](variables.tf#L79) | PubSub topic name. | string
| ✓ | |
-| [project_id](variables.tf#L84) | Project used for resources. | string
| ✓ | |
+| [name](variables.tf#L94) | PubSub topic name. | string
| ✓ | |
+| [project_id](variables.tf#L99) | Project used for resources. | string
| ✓ | |
| [bigquery_subscription_configs](variables.tf#L17) | Configuration parameters for BigQuery subscriptions. | map(object({…}))
| | {}
|
-| [dead_letter_configs](variables.tf#L28) | Per-subscription dead letter policy configuration. | map(object({…}))
| | {}
|
-| [defaults](variables.tf#L37) | Subscription defaults for options. | object({…})
| | {…}
|
-| [iam](variables.tf#L55) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
-| [kms_key](variables.tf#L61) | KMS customer managed encryption key. | string
| | null
|
-| [labels](variables.tf#L67) | Labels. | map(string)
| | {}
|
-| [message_retention_duration](variables.tf#L73) | Minimum duration to retain a message after it is published to the topic. | string
| | null
|
-| [push_configs](variables.tf#L89) | Push subscription configurations. | map(object({…}))
| | {}
|
-| [regions](variables.tf#L102) | List of regions used to set persistence policy. | list(string)
| | []
|
-| [schema](variables.tf#L108) | Topic schema. If set, all messages in this topic should follow this schema. | object({…})
| | null
|
-| [subscription_iam](variables.tf#L118) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string)))
| | {}
|
-| [subscriptions](variables.tf#L124) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…}))
| | {}
|
+| [cloud_storage_subscription_configs](variables.tf#L28) | Configuration parameters for Cloud Storage subscriptions. | map(object({…}))
| | {}
|
+| [dead_letter_configs](variables.tf#L43) | Per-subscription dead letter policy configuration. | map(object({…}))
| | {}
|
+| [defaults](variables.tf#L52) | Subscription defaults for options. | object({…})
| | {…}
|
+| [iam](variables.tf#L70) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
+| [kms_key](variables.tf#L76) | KMS customer managed encryption key. | string
| | null
|
+| [labels](variables.tf#L82) | Labels. | map(string)
| | {}
|
+| [message_retention_duration](variables.tf#L88) | Minimum duration to retain a message after it is published to the topic. | string
| | null
|
+| [push_configs](variables.tf#L104) | Push subscription configurations. | map(object({…}))
| | {}
|
+| [regions](variables.tf#L117) | List of regions used to set persistence policy. | list(string)
| | []
|
+| [schema](variables.tf#L123) | Topic schema. If set, all messages in this topic should follow this schema. | object({…})
| | null
|
+| [subscription_iam](variables.tf#L133) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string)))
| | {}
|
+| [subscriptions](variables.tf#L139) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…}))
| | {}
|
## Outputs
@@ -174,5 +201,4 @@ module "pubsub" {
| [subscription_id](outputs.tf#L36) | Subscription ids. | |
| [subscriptions](outputs.tf#L46) | Subscription resources. | |
| [topic](outputs.tf#L54) | Topic resource. | |
-
diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf
index ccb6f5d7..2a7f873f 100644
--- a/modules/pubsub/main.tf
+++ b/modules/pubsub/main.tf
@@ -24,6 +24,9 @@ locals {
}
]
])
+ metadata_config = {
+ for k, v in var.cloud_storage_subscription_configs : k => v.avro_config
+ }
oidc_config = {
for k, v in var.push_configs : k => v.oidc_token
}
@@ -127,6 +130,25 @@ resource "google_pubsub_subscription" "default" {
drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields
}
}
+
+ dynamic "cloud_storage_config" {
+ for_each = try(var.cloud_storage_subscription_configs[each.key], null) == null ? [] : [""]
+ content {
+ bucket = var.cloud_storage_subscription_configs[each.key].bucket
+ filename_prefix = var.cloud_storage_subscription_configs[each.key].filename_prefix
+ filename_suffix = var.cloud_storage_subscription_configs[each.key].filename_suffix
+ max_duration = var.cloud_storage_subscription_configs[each.key].max_duration
+ max_bytes = var.cloud_storage_subscription_configs[each.key].max_bytes
+ dynamic "avro_config" {
+ for_each = (
+ local.metadata_config[each.key] == null ? [] : [""]
+ )
+ content {
+ write_metadata = local.metadata_config[each.key].write_metadata
+ }
+ }
+ }
+ }
}
resource "google_pubsub_subscription_iam_binding" "default" {
diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf
index afefb4a8..35dfc172 100644
--- a/modules/pubsub/variables.tf
+++ b/modules/pubsub/variables.tf
@@ -25,6 +25,21 @@ variable "bigquery_subscription_configs" {
default = {}
}
+variable "cloud_storage_subscription_configs" {
+ description = "Configuration parameters for Cloud Storage subscriptions."
+ type = map(object({
+ bucket = string
+ filename_prefix = string
+ filename_suffix = string
+ max_duration = string
+ max_bytes = number
+ avro_config = object({
+ write_metadata = bool
+ })
+ }))
+ default = {}
+}
+
variable "dead_letter_configs" {
description = "Per-subscription dead letter policy configuration."
type = map(object({