Added possibility to use gcs push endpoint on pubsub subscription
This commit is contained in:
Luigi Bitonti
parent
95a3752cc7
commit
f2fc67d72d
|
|
@ -124,6 +124,33 @@ module "pubsub" {
|
|||
# tftest modules=1 resources=2
|
||||
```
|
||||
|
||||
### Cloud Storage subscriptions
|
||||
|
||||
Cloud Storage subscriptions need extra configuration in the `cloud_storage_subscription_configs` variable.
|
||||
|
||||
```hcl
|
||||
module "pubsub" {
|
||||
source = "./fabric/modules/pubsub"
|
||||
pproject_id = "my-project"
|
||||
name = "my-topic"
|
||||
subscriptions = {
|
||||
test-cloudstorage = null
|
||||
}
|
||||
cloud_storage_subscription_configs = {
|
||||
test-cloudstorage = {
|
||||
bucket = "my-bucket"
|
||||
filename_prefix = "test_prefix"
|
||||
filename_suffix = "test_suffix"
|
||||
max_duration = "100s"
|
||||
max_bytes = 1000
|
||||
avro_config = {
|
||||
write_metadata = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=2
|
||||
```
|
||||
### Subscriptions with IAM
|
||||
|
||||
```hcl
|
||||
|
|
@ -144,25 +171,25 @@ module "pubsub" {
|
|||
# tftest modules=1 resources=3
|
||||
```
|
||||
<!-- BEGIN TFDOC -->
|
||||
|
||||
## Variables
|
||||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [name](variables.tf#L79) | PubSub topic name. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L84) | Project used for resources. | <code>string</code> | ✓ | |
|
||||
| [name](variables.tf#L94) | PubSub topic name. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L99) | Project used for resources. | <code>string</code> | ✓ | |
|
||||
| [bigquery_subscription_configs](variables.tf#L17) | Configuration parameters for BigQuery subscriptions. | <code title="map(object({ table = string use_topic_schema = bool write_metadata = bool drop_unknown_fields = bool }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [dead_letter_configs](variables.tf#L28) | Per-subscription dead letter policy configuration. | <code title="map(object({ topic = string max_delivery_attempts = number }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [defaults](variables.tf#L37) | Subscription defaults for options. | <code title="object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string })">object({…})</code> | | <code title="{ ack_deadline_seconds = null message_retention_duration = null retain_acked_messages = null expiration_policy_ttl = null filter = null }">{…}</code> |
|
||||
| [iam](variables.tf#L55) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [kms_key](variables.tf#L61) | KMS customer managed encryption key. | <code>string</code> | | <code>null</code> |
|
||||
| [labels](variables.tf#L67) | Labels. | <code>map(string)</code> | | <code>{}</code> |
|
||||
| [message_retention_duration](variables.tf#L73) | Minimum duration to retain a message after it is published to the topic. | <code>string</code> | | <code>null</code> |
|
||||
| [push_configs](variables.tf#L89) | Push subscription configurations. | <code title="map(object({ attributes = map(string) endpoint = string oidc_token = object({ audience = string service_account_email = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [regions](variables.tf#L102) | List of regions used to set persistence policy. | <code>list(string)</code> | | <code>[]</code> |
|
||||
| [schema](variables.tf#L108) | Topic schema. If set, all messages in this topic should follow this schema. | <code title="object({ definition = string msg_encoding = optional(string, "ENCODING_UNSPECIFIED") schema_type = string })">object({…})</code> | | <code>null</code> |
|
||||
| [subscription_iam](variables.tf#L118) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code>{}</code> |
|
||||
| [subscriptions](variables.tf#L124) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map(object({ labels = map(string) options = object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [cloud_storage_subscription_configs](variables.tf#L28) | Configuration parameters for Cloud Storage subscriptions. | <code title="map(object({ bucket = string filename_prefix = string filename_suffix = string max_duration = string max_bytes = number avro_config = object({ write_metadata = bool }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [dead_letter_configs](variables.tf#L43) | Per-subscription dead letter policy configuration. | <code title="map(object({ topic = string max_delivery_attempts = number }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [defaults](variables.tf#L52) | Subscription defaults for options. | <code title="object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string })">object({…})</code> | | <code title="{ ack_deadline_seconds = null message_retention_duration = null retain_acked_messages = null expiration_policy_ttl = null filter = null }">{…}</code> |
|
||||
| [iam](variables.tf#L70) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [kms_key](variables.tf#L76) | KMS customer managed encryption key. | <code>string</code> | | <code>null</code> |
|
||||
| [labels](variables.tf#L82) | Labels. | <code>map(string)</code> | | <code>{}</code> |
|
||||
| [message_retention_duration](variables.tf#L88) | Minimum duration to retain a message after it is published to the topic. | <code>string</code> | | <code>null</code> |
|
||||
| [push_configs](variables.tf#L104) | Push subscription configurations. | <code title="map(object({ attributes = map(string) endpoint = string oidc_token = object({ audience = string service_account_email = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [regions](variables.tf#L117) | List of regions used to set persistence policy. | <code>list(string)</code> | | <code>[]</code> |
|
||||
| [schema](variables.tf#L123) | Topic schema. If set, all messages in this topic should follow this schema. | <code title="object({ definition = string msg_encoding = optional(string, "ENCODING_UNSPECIFIED") schema_type = string })">object({…})</code> | | <code>null</code> |
|
||||
| [subscription_iam](variables.tf#L133) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code>{}</code> |
|
||||
| [subscriptions](variables.tf#L139) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map(object({ labels = map(string) options = object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
@ -174,5 +201,4 @@ module "pubsub" {
|
|||
| [subscription_id](outputs.tf#L36) | Subscription ids. | |
|
||||
| [subscriptions](outputs.tf#L46) | Subscription resources. | |
|
||||
| [topic](outputs.tf#L54) | Topic resource. | |
|
||||
|
||||
<!-- END TFDOC -->
|
||||
|
|
|
|||
|
|
@ -24,6 +24,9 @@ locals {
|
|||
}
|
||||
]
|
||||
])
|
||||
metadata_config = {
|
||||
for k, v in var.cloud_storage_subscription_configs : k => v.avro_config
|
||||
}
|
||||
oidc_config = {
|
||||
for k, v in var.push_configs : k => v.oidc_token
|
||||
}
|
||||
|
|
@ -127,6 +130,25 @@ resource "google_pubsub_subscription" "default" {
|
|||
drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields
|
||||
}
|
||||
}
|
||||
|
||||
dynamic "cloud_storage_config" {
|
||||
for_each = try(var.cloud_storage_subscription_configs[each.key], null) == null ? [] : [""]
|
||||
content {
|
||||
bucket = var.cloud_storage_subscription_configs[each.key].bucket
|
||||
filename_prefix = var.cloud_storage_subscription_configs[each.key].filename_prefix
|
||||
filename_suffix = var.cloud_storage_subscription_configs[each.key].filename_suffix
|
||||
max_duration = var.cloud_storage_subscription_configs[each.key].max_duration
|
||||
max_bytes = var.cloud_storage_subscription_configs[each.key].max_bytes
|
||||
dynamic "avro_config" {
|
||||
for_each = (
|
||||
local.metadata_config[each.key] == null ? [] : [""]
|
||||
)
|
||||
content {
|
||||
write_metadata = local.metadata_config[each.key].write_metadata
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_pubsub_subscription_iam_binding" "default" {
|
||||
|
|
|
|||
|
|
@ -25,6 +25,21 @@ variable "bigquery_subscription_configs" {
|
|||
default = {}
|
||||
}
|
||||
|
||||
variable "cloud_storage_subscription_configs" {
|
||||
description = "Configuration parameters for Cloud Storage subscriptions."
|
||||
type = map(object({
|
||||
bucket = string
|
||||
filename_prefix = string
|
||||
filename_suffix = string
|
||||
max_duration = string
|
||||
max_bytes = number
|
||||
avro_config = object({
|
||||
write_metadata = bool
|
||||
})
|
||||
}))
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "dead_letter_configs" {
|
||||
description = "Per-subscription dead letter policy configuration."
|
||||
type = map(object({
|
||||
|
|
|
|||
Loading…
Reference in New Issue