Merge pull request #997 from iht/bigquery_subscriptions
Add BigQuery subcriptions to Pubsub module.
This commit is contained in:
commit
3651231fdc
|
@ -2,7 +2,6 @@
|
||||||
|
|
||||||
This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels, as well as schemas.
|
This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels, as well as schemas.
|
||||||
|
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
### Simple topic with IAM
|
### Simple topic with IAM
|
||||||
|
@ -101,6 +100,30 @@ module "pubsub" {
|
||||||
# tftest modules=1 resources=2
|
# tftest modules=1 resources=2
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### BigQuery subscriptions
|
||||||
|
|
||||||
|
BigQuery subscriptions need extra configuration in the `bigquery_subscription_configs` variable.
|
||||||
|
|
||||||
|
```hcl
|
||||||
|
module "pubsub" {
|
||||||
|
source = "./fabric/modules/pubsub"
|
||||||
|
project_id = "my-project"
|
||||||
|
name = "my-topic"
|
||||||
|
subscriptions = {
|
||||||
|
test-bigquery = null
|
||||||
|
}
|
||||||
|
bigquery_subscription_configs = {
|
||||||
|
test-bigquery = {
|
||||||
|
table = "my_project_id:my_dataset.my_table"
|
||||||
|
use_topic_schema = true
|
||||||
|
write_metadata = false
|
||||||
|
drop_unknown_fields = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# tftest modules=1 resources=2
|
||||||
|
```
|
||||||
|
|
||||||
### Subscriptions with IAM
|
### Subscriptions with IAM
|
||||||
|
|
||||||
```hcl
|
```hcl
|
||||||
|
@ -126,19 +149,20 @@ module "pubsub" {
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [name](variables.tf#L68) | PubSub topic name. | <code>string</code> | ✓ | |
|
| [name](variables.tf#L79) | PubSub topic name. | <code>string</code> | ✓ | |
|
||||||
| [project_id](variables.tf#L73) | Project used for resources. | <code>string</code> | ✓ | |
|
| [project_id](variables.tf#L84) | Project used for resources. | <code>string</code> | ✓ | |
|
||||||
| [dead_letter_configs](variables.tf#L17) | Per-subscription dead letter policy configuration. | <code title="map(object({ topic = string max_delivery_attempts = number }))">map(object({…}))</code> | | <code>{}</code> |
|
| [bigquery_subscription_configs](variables.tf#L17) | Configuration parameters for BigQuery subscriptions. | <code title="map(object({ table = string use_topic_schema = bool write_metadata = bool drop_unknown_fields = bool }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [defaults](variables.tf#L26) | Subscription defaults for options. | <code title="object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string })">object({…})</code> | | <code title="{ ack_deadline_seconds = null message_retention_duration = null retain_acked_messages = null expiration_policy_ttl = null filter = null }">{…}</code> |
|
| [dead_letter_configs](variables.tf#L28) | Per-subscription dead letter policy configuration. | <code title="map(object({ topic = string max_delivery_attempts = number }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [iam](variables.tf#L44) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
| [defaults](variables.tf#L37) | Subscription defaults for options. | <code title="object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string })">object({…})</code> | | <code title="{ ack_deadline_seconds = null message_retention_duration = null retain_acked_messages = null expiration_policy_ttl = null filter = null }">{…}</code> |
|
||||||
| [kms_key](variables.tf#L50) | KMS customer managed encryption key. | <code>string</code> | | <code>null</code> |
|
| [iam](variables.tf#L55) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||||
| [labels](variables.tf#L56) | Labels. | <code>map(string)</code> | | <code>{}</code> |
|
| [kms_key](variables.tf#L61) | KMS customer managed encryption key. | <code>string</code> | | <code>null</code> |
|
||||||
| [message_retention_duration](variables.tf#L62) | Minimum duration to retain a message after it is published to the topic. | <code>string</code> | | <code>null</code> |
|
| [labels](variables.tf#L67) | Labels. | <code>map(string)</code> | | <code>{}</code> |
|
||||||
| [push_configs](variables.tf#L78) | Push subscription configurations. | <code title="map(object({ attributes = map(string) endpoint = string oidc_token = object({ audience = string service_account_email = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
| [message_retention_duration](variables.tf#L73) | Minimum duration to retain a message after it is published to the topic. | <code>string</code> | | <code>null</code> |
|
||||||
| [regions](variables.tf#L91) | List of regions used to set persistence policy. | <code>list(string)</code> | | <code>[]</code> |
|
| [push_configs](variables.tf#L89) | Push subscription configurations. | <code title="map(object({ attributes = map(string) endpoint = string oidc_token = object({ audience = string service_account_email = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [schema](variables.tf#L97) | Topic schema. If set, all messages in this topic should follow this schema. | <code title="object({ definition = string msg_encoding = optional(string, "ENCODING_UNSPECIFIED") schema_type = string })">object({…})</code> | | <code>null</code> |
|
| [regions](variables.tf#L102) | List of regions used to set persistence policy. | <code>list(string)</code> | | <code>[]</code> |
|
||||||
| [subscription_iam](variables.tf#L107) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code>{}</code> |
|
| [schema](variables.tf#L108) | Topic schema. If set, all messages in this topic should follow this schema. | <code title="object({ definition = string msg_encoding = optional(string, "ENCODING_UNSPECIFIED") schema_type = string })">object({…})</code> | | <code>null</code> |
|
||||||
| [subscriptions](variables.tf#L113) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map(object({ labels = map(string) options = object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
| [subscription_iam](variables.tf#L118) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code>{}</code> |
|
||||||
|
| [subscriptions](variables.tf#L124) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map(object({ labels = map(string) options = object({ ack_deadline_seconds = number message_retention_duration = string retain_acked_messages = bool expiration_policy_ttl = string filter = string }) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
|
|
|
@ -116,6 +116,16 @@ resource "google_pubsub_subscription" "default" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
dynamic "bigquery_config" {
|
||||||
|
for_each = try(var.bigquery_subscription_configs[each.key], null) == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
table = var.bigquery_subscription_configs[each.key].table
|
||||||
|
use_topic_schema = var.bigquery_subscription_configs[each.key].use_topic_schema
|
||||||
|
write_metadata = var.bigquery_subscription_configs[each.key].write_metadata
|
||||||
|
drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_pubsub_subscription_iam_binding" "default" {
|
resource "google_pubsub_subscription_iam_binding" "default" {
|
||||||
|
|
|
@ -14,6 +14,17 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
variable "bigquery_subscription_configs" {
|
||||||
|
description = "Configuration parameters for BigQuery subscriptions."
|
||||||
|
type = map(object({
|
||||||
|
table = string
|
||||||
|
use_topic_schema = bool
|
||||||
|
write_metadata = bool
|
||||||
|
drop_unknown_fields = bool
|
||||||
|
}))
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
variable "dead_letter_configs" {
|
variable "dead_letter_configs" {
|
||||||
description = "Per-subscription dead letter policy configuration."
|
description = "Per-subscription dead letter policy configuration."
|
||||||
type = map(object({
|
type = map(object({
|
||||||
|
|
Loading…
Reference in New Issue