Added possibility to use gcs push endpoint on pubsub subscription
This commit is contained in:
parent
95a3752cc7
commit
f2fc67d72d
|
@ -124,6 +124,33 @@ module "pubsub" {
|
||||||
# tftest modules=1 resources=2
|
# tftest modules=1 resources=2
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Cloud Storage subscriptions
|
||||||
|
|
||||||
|
Cloud Storage subscriptions need extra configuration in the `cloud_storage_subscription_configs` variable.
|
||||||
|
|
||||||
|
```hcl
|
||||||
|
module "pubsub" {
|
||||||
|
source = "./fabric/modules/pubsub"
|
||||||
|
pproject_id = "my-project"
|
||||||
|
name = "my-topic"
|
||||||
|
subscriptions = {
|
||||||
|
test-cloudstorage = null
|
||||||
|
}
|
||||||
|
cloud_storage_subscription_configs = {
|
||||||
|
test-cloudstorage = {
|
||||||
|
bucket = "my-bucket"
|
||||||
|
filename_prefix = "test_prefix"
|
||||||
|
filename_suffix = "test_suffix"
|
||||||
|
max_duration = "100s"
|
||||||
|
max_bytes = 1000
|
||||||
|
avro_config = {
|
||||||
|
write_metadata = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# tftest modules=1 resources=2
|
||||||
|
```
|
||||||
### Subscriptions with IAM
|
### Subscriptions with IAM
|
||||||
|
|
||||||
```hcl
|
```hcl
|
||||||
|
@ -144,25 +171,25 @@ module "pubsub" {
|
||||||
# tftest modules=1 resources=3
|
# tftest modules=1 resources=3
|
||||||
```
|
```
|
||||||
<!-- BEGIN TFDOC -->
|
<!-- BEGIN TFDOC -->
|
||||||
|
|
||||||
## Variables
|
## Variables
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [name](variables.tf#L79) | PubSub topic name. | <code>string</code> | ✓ | |
|
| [name](variables.tf#L94) | PubSub topic name. | <code>string</code> | ✓ | |
|
||||||
| [project_id](variables.tf#L84) | Project used for resources. | <code>string</code> | ✓ | |
|
| [project_id](variables.tf#L99) | Project used for resources. | <code>string</code> | ✓ | |
|
||||||
| [bigquery_subscription_configs](variables.tf#L17) | Configuration parameters for BigQuery subscriptions. | <code title="map(object({ table = string use_topic_schema = bool write_metadata = bool drop_unknown_fields = bool }))">map(object({…}))</code> | | <code>{}</code> |
|
| [bigquery_subscription_configs](variables.tf#L17) | Configuration parameters for BigQuery subscriptions. | <code title="map(object({ table = string use_topic_schema = bool write_metadata = bool drop_unknown_fields = bool }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [dead_letter_configs](variables.tf#L28) | Per-subscription dead letter policy configuration. | <code title="map(object({ topic = string max_delivery_attempts = number }))">map(object({…}))</code> | | <code>{}</code> |
|
| [cloud_storage_subscription_configs](variables.tf#L28) | Configuration parameters for Cloud Storage subscriptions. | <code title="map(object({ bucket = string filename_prefix = string filename_suffix = string max_duration = string max_bytes = number avro_config = object({ write_metadata = bool }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [defaults](variables.tf#L37) | Subscription defaults for options. | <code title="object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string })">object({…})</code> | | <code title="{ ack_deadline_seconds = null message_retention_duration = null retain_acked_messages = null expiration_policy_ttl = null filter = null }">{…}</code> |
|
| [dead_letter_configs](variables.tf#L43) | Per-subscription dead letter policy configuration. | <code title="map(object({ topic = string max_delivery_attempts = number }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [iam](variables.tf#L55) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
| [defaults](variables.tf#L52) | Subscription defaults for options. | <code title="object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string })">object({…})</code> | | <code title="{ ack_deadline_seconds = null message_retention_duration = null retain_acked_messages = null expiration_policy_ttl = null filter = null }">{…}</code> |
|
||||||
| [kms_key](variables.tf#L61) | KMS customer managed encryption key. | <code>string</code> | | <code>null</code> |
|
| [iam](variables.tf#L70) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||||
| [labels](variables.tf#L67) | Labels. | <code>map(string)</code> | | <code>{}</code> |
|
| [kms_key](variables.tf#L76) | KMS customer managed encryption key. | <code>string</code> | | <code>null</code> |
|
||||||
| [message_retention_duration](variables.tf#L73) | Minimum duration to retain a message after it is published to the topic. | <code>string</code> | | <code>null</code> |
|
| [labels](variables.tf#L82) | Labels. | <code>map(string)</code> | | <code>{}</code> |
|
||||||
| [push_configs](variables.tf#L89) | Push subscription configurations. | <code title="map(object({ attributes = map(string) endpoint = string oidc_token = object({ audience = string service_account_email = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
| [message_retention_duration](variables.tf#L88) | Minimum duration to retain a message after it is published to the topic. | <code>string</code> | | <code>null</code> |
|
||||||
| [regions](variables.tf#L102) | List of regions used to set persistence policy. | <code>list(string)</code> | | <code>[]</code> |
|
| [push_configs](variables.tf#L104) | Push subscription configurations. | <code title="map(object({ attributes = map(string) endpoint = string oidc_token = object({ audience = string service_account_email = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [schema](variables.tf#L108) | Topic schema. If set, all messages in this topic should follow this schema. | <code title="object({ definition = string msg_encoding = optional(string, "ENCODING_UNSPECIFIED") schema_type = string })">object({…})</code> | | <code>null</code> |
|
| [regions](variables.tf#L117) | List of regions used to set persistence policy. | <code>list(string)</code> | | <code>[]</code> |
|
||||||
| [subscription_iam](variables.tf#L118) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code>{}</code> |
|
| [schema](variables.tf#L123) | Topic schema. If set, all messages in this topic should follow this schema. | <code title="object({ definition = string msg_encoding = optional(string, "ENCODING_UNSPECIFIED") schema_type = string })">object({…})</code> | | <code>null</code> |
|
||||||
| [subscriptions](variables.tf#L124) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map(object({ labels = map(string) options = object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
| [subscription_iam](variables.tf#L133) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code>{}</code> |
|
||||||
|
| [subscriptions](variables.tf#L139) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map(object({ labels = map(string) options = object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
|
@ -174,5 +201,4 @@ module "pubsub" {
|
||||||
| [subscription_id](outputs.tf#L36) | Subscription ids. | |
|
| [subscription_id](outputs.tf#L36) | Subscription ids. | |
|
||||||
| [subscriptions](outputs.tf#L46) | Subscription resources. | |
|
| [subscriptions](outputs.tf#L46) | Subscription resources. | |
|
||||||
| [topic](outputs.tf#L54) | Topic resource. | |
|
| [topic](outputs.tf#L54) | Topic resource. | |
|
||||||
|
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
|
@ -24,6 +24,9 @@ locals {
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
])
|
])
|
||||||
|
metadata_config = {
|
||||||
|
for k, v in var.cloud_storage_subscription_configs : k => v.avro_config
|
||||||
|
}
|
||||||
oidc_config = {
|
oidc_config = {
|
||||||
for k, v in var.push_configs : k => v.oidc_token
|
for k, v in var.push_configs : k => v.oidc_token
|
||||||
}
|
}
|
||||||
|
@ -127,6 +130,25 @@ resource "google_pubsub_subscription" "default" {
|
||||||
drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields
|
drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
dynamic "cloud_storage_config" {
|
||||||
|
for_each = try(var.cloud_storage_subscription_configs[each.key], null) == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
bucket = var.cloud_storage_subscription_configs[each.key].bucket
|
||||||
|
filename_prefix = var.cloud_storage_subscription_configs[each.key].filename_prefix
|
||||||
|
filename_suffix = var.cloud_storage_subscription_configs[each.key].filename_suffix
|
||||||
|
max_duration = var.cloud_storage_subscription_configs[each.key].max_duration
|
||||||
|
max_bytes = var.cloud_storage_subscription_configs[each.key].max_bytes
|
||||||
|
dynamic "avro_config" {
|
||||||
|
for_each = (
|
||||||
|
local.metadata_config[each.key] == null ? [] : [""]
|
||||||
|
)
|
||||||
|
content {
|
||||||
|
write_metadata = local.metadata_config[each.key].write_metadata
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_pubsub_subscription_iam_binding" "default" {
|
resource "google_pubsub_subscription_iam_binding" "default" {
|
||||||
|
|
|
@ -25,6 +25,21 @@ variable "bigquery_subscription_configs" {
|
||||||
default = {}
|
default = {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "cloud_storage_subscription_configs" {
|
||||||
|
description = "Configuration parameters for Cloud Storage subscriptions."
|
||||||
|
type = map(object({
|
||||||
|
bucket = string
|
||||||
|
filename_prefix = string
|
||||||
|
filename_suffix = string
|
||||||
|
max_duration = string
|
||||||
|
max_bytes = number
|
||||||
|
avro_config = object({
|
||||||
|
write_metadata = bool
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
variable "dead_letter_configs" {
|
variable "dead_letter_configs" {
|
||||||
description = "Per-subscription dead letter policy configuration."
|
description = "Per-subscription dead letter policy configuration."
|
||||||
type = map(object({
|
type = map(object({
|
||||||
|
|
Loading…
Reference in New Issue