diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 8dd9e5ff..6e706d9e 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -61,16 +61,10 @@ module "pubsub" { project_id = "my-project" name = "my-topic" subscriptions = { - test-pull = null + test-pull = {} test-pull-override = { - labels = { test = "override" } - options = { - ack_deadline_seconds = null - message_retention_duration = null - retain_acked_messages = true - expiration_policy_ttl = null - filter = null - } + labels = { test = "override" } + retain_acked_messages = true } } } @@ -87,13 +81,10 @@ module "pubsub" { project_id = "my-project" name = "my-topic" subscriptions = { - test-push = null - } - push_configs = { test-push = { - endpoint = "https://example.com/foo" - attributes = null - oidc_token = null + push = { + endpoint = "https://example.com/foo" + } } } } @@ -110,14 +101,13 @@ module "pubsub" { project_id = "my-project" name = "my-topic" subscriptions = { - test-bigquery = null - } - bigquery_subscription_configs = { test-bigquery = { - table = "my_project_id:my_dataset.my_table" - use_topic_schema = true - write_metadata = false - drop_unknown_fields = true + bigquery = { + table = "my_project_id:my_dataset.my_table" + use_topic_schema = true + write_metadata = false + drop_unknown_fields = true + } } } } @@ -134,17 +124,16 @@ module "pubsub" { project_id = "my-project" name = "my-topic" subscriptions = { - test-cloudstorage = null - } - cloud_storage_subscription_configs = { test-cloudstorage = { - bucket = "my-bucket" - filename_prefix = "test_prefix" - filename_suffix = "test_suffix" - max_duration = "100s" - max_bytes = 1000 - avro_config = { - write_metadata = true + cloud_storage = { + bucket = "my-bucket" + filename_prefix = "test_prefix" + filename_suffix = "test_suffix" + max_duration = "100s" + max_bytes = 1000 + avro_config = { + write_metadata = true + } } } } @@ -159,12 +148,10 @@ module "pubsub" { project_id = "my-project" name = "my-topic" subscriptions = { - test-1 = null - test-1 = null - } - subscription_iam = { test-1 = { - "roles/pubsub.subscriber" = ["user:user1@ludomagno.net"] + iam = { + "roles/pubsub.subscriber" = ["user:user1@example.com"] + } } } } diff --git a/modules/pubsub/iam.tf b/modules/pubsub/iam.tf new file mode 100644 index 00000000..4e39b43a --- /dev/null +++ b/modules/pubsub/iam.tf @@ -0,0 +1,140 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the authoritative. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + subscription_iam = flatten([ + for k, v in var.subscriptions : [ + for role, members in v.iam : { + subscription = k + role = role + members = members + } + ] + ]) + subscription_iam_bindings = merge([ + for k, v in var.subscriptions : { + for binding_key, data in v.iam_bindings : + binding_key => { + subscription = k + role = data.role + members = data.members + condition = data.condition + } + } + ]...) + subscription_iam_bindings_additive = merge([ + for k, v in var.subscriptions : { + for binding_key, data in v.iam_bindings_additive : + binding_key => { + subscription = k + role = data.role + member = data.member + condition = data.condition + } + } + ]...) +} + +moved { + from = google_pubsub_topic_iam_binding.default + to = google_pubsub_topic_iam_binding.authoritative +} + +resource "google_pubsub_topic_iam_binding" "authoritative" { + for_each = var.iam + project = var.project_id + topic = google_pubsub_topic.default.name + role = each.key + members = each.value +} + +resource "google_pubsub_topic_iam_binding" "bindings" { + for_each = var.iam_bindings + topic = google_pubsub_topic.default.name + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_pubsub_topic_iam_member" "bindings" { + for_each = var.iam_bindings_additive + topic = google_pubsub_topic.default.name + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +moved { + from = google_pubsub_subscription_iam_binding.default + to = google_pubsub_subscription_iam_binding.authoritative +} + +resource "google_pubsub_subscription_iam_binding" "authoritative" { + for_each = { + for binding in local.subscription_iam : + "${binding.subscription}.${binding.role}" => binding + } + project = var.project_id + subscription = google_pubsub_subscription.default[each.value.subscription].name + role = each.value.role + members = each.value.members +} + +resource "google_pubsub_subscription_iam_binding" "bindings" { + for_each = local.subscription_iam_bindings + project = var.project_id + subscription = google_pubsub_subscription.default[each.value.subscription].name + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_pubsub_subscription_iam_member" "members" { + for_each = local.subscription_iam_bindings_additive + project = var.project_id + subscription = google_pubsub_subscription.default[each.value.subscription].name + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index 2a7f873f..de065029 100644 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,27 +15,6 @@ */ locals { - sub_iam_members = flatten([ - for sub, roles in var.subscription_iam : [ - for role, members in roles : { - sub = sub - role = role - members = members - } - ] - ]) - metadata_config = { - for k, v in var.cloud_storage_subscription_configs : k => v.avro_config - } - oidc_config = { - for k, v in var.push_configs : k => v.oidc_token - } - subscriptions = { - for k, v in var.subscriptions : k => { - labels = try(v.labels, v, null) == null ? var.labels : v.labels - options = try(v.options, v, null) == null ? var.defaults : v.options - } - } topic_id_static = "projects/${var.project_id}/topics/${var.name}" } @@ -70,94 +49,73 @@ resource "google_pubsub_topic" "default" { } } -resource "google_pubsub_topic_iam_binding" "default" { - for_each = var.iam - project = var.project_id - topic = google_pubsub_topic.default.name - role = each.key - members = each.value -} - resource "google_pubsub_subscription" "default" { - for_each = local.subscriptions - project = var.project_id - name = each.key - topic = google_pubsub_topic.default.name - labels = each.value.labels - ack_deadline_seconds = each.value.options.ack_deadline_seconds - message_retention_duration = each.value.options.message_retention_duration - retain_acked_messages = each.value.options.retain_acked_messages - filter = each.value.options.filter + for_each = var.subscriptions + project = var.project_id + name = each.key + topic = google_pubsub_topic.default.name + labels = each.value.labels + ack_deadline_seconds = each.value.ack_deadline_seconds + message_retention_duration = each.value.message_retention_duration + retain_acked_messages = each.value.retain_acked_messages + filter = each.value.filter + enable_message_ordering = each.value.enable_message_ordering + enable_exactly_once_delivery = each.value.enable_exactly_once_delivery dynamic "expiration_policy" { - for_each = each.value.options.expiration_policy_ttl == null ? [] : [""] + for_each = each.value.expiration_policy_ttl == null ? [] : [""] content { - ttl = each.value.options.expiration_policy_ttl + ttl = each.value.expiration_policy_ttl } } dynamic "dead_letter_policy" { - for_each = try(var.dead_letter_configs[each.key], null) == null ? [] : [""] + for_each = each.value.dead_letter_policy == null ? [] : [""] content { - dead_letter_topic = var.dead_letter_configs[each.key].topic - max_delivery_attempts = var.dead_letter_configs[each.key].max_delivery_attempts + dead_letter_topic = each.value.dead_letter_policy.topic + max_delivery_attempts = each.value.dead_letter_policy.max_delivery_attempts } } dynamic "push_config" { - for_each = try(var.push_configs[each.key], null) == null ? [] : [""] + for_each = each.value.push == null ? [] : [""] content { - push_endpoint = var.push_configs[each.key].endpoint - attributes = var.push_configs[each.key].attributes + push_endpoint = each.value.push.endpoint + attributes = each.value.push.attributes dynamic "oidc_token" { - for_each = ( - local.oidc_config[each.key] == null ? [] : [""] - ) + for_each = each.value.push.oidc_token == null ? [] : [""] content { - service_account_email = local.oidc_config[each.key].service_account_email - audience = local.oidc_config[each.key].audience + service_account_email = each.value.push.oidc_token.service_account_email + audience = each.value.push.oidc_token.audience } } } } dynamic "bigquery_config" { - for_each = try(var.bigquery_subscription_configs[each.key], null) == null ? [] : [""] + for_each = each.value.bigquery == null ? [] : [""] content { - table = var.bigquery_subscription_configs[each.key].table - use_topic_schema = var.bigquery_subscription_configs[each.key].use_topic_schema - write_metadata = var.bigquery_subscription_configs[each.key].write_metadata - drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields + table = each.value.bigquery.table + use_topic_schema = each.value.bigquery.use_topic_schema + write_metadata = each.value.bigquery.write_metadata + drop_unknown_fields = each.value.bigquery.drop_unknown_fields } } dynamic "cloud_storage_config" { - for_each = try(var.cloud_storage_subscription_configs[each.key], null) == null ? [] : [""] + for_each = each.value.cloud_storage == null ? [] : [""] content { - bucket = var.cloud_storage_subscription_configs[each.key].bucket - filename_prefix = var.cloud_storage_subscription_configs[each.key].filename_prefix - filename_suffix = var.cloud_storage_subscription_configs[each.key].filename_suffix - max_duration = var.cloud_storage_subscription_configs[each.key].max_duration - max_bytes = var.cloud_storage_subscription_configs[each.key].max_bytes + bucket = each.value.cloud_storage.bucket + filename_prefix = each.value.cloud_storage.filename_prefix + filename_suffix = each.value.cloud_storage.filename_suffix + max_duration = each.value.cloud_storage.max_duration + max_bytes = each.value.cloud_storage.max_bytes dynamic "avro_config" { - for_each = ( - local.metadata_config[each.key] == null ? [] : [""] - ) + for_each = each.value.cloud_storage.avro_config == null ? [] : [""] content { - write_metadata = local.metadata_config[each.key].write_metadata + write_metadata = each.value.cloud_storage.avro_config.write_metadata } } } } } - -resource "google_pubsub_subscription_iam_binding" "default" { - for_each = { - for binding in local.sub_iam_members : - "${binding.sub}.${binding.role}" => binding - } - project = var.project_id - subscription = google_pubsub_subscription.default[each.value.sub].name - role = each.value.role - members = each.value.members -} diff --git a/modules/pubsub/outputs.tf b/modules/pubsub/outputs.tf index 0d149302..8218e2b3 100644 --- a/modules/pubsub/outputs.tf +++ b/modules/pubsub/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,8 @@ output "id" { value = local.topic_id_static depends_on = [ google_pubsub_topic.default, - google_pubsub_topic_iam_binding.default + google_pubsub_topic_iam_binding.authoritative, + google_pubsub_topic_iam_binding.bindings ] } @@ -39,7 +40,8 @@ output "subscription_id" { for k, v in google_pubsub_subscription.default : k => v.id } depends_on = [ - google_pubsub_subscription_iam_binding.default + google_pubsub_subscription_iam_binding.authoritative, + google_pubsub_subscription_iam_binding.bindings ] } @@ -47,7 +49,8 @@ output "subscriptions" { description = "Subscription resources." value = google_pubsub_subscription.default depends_on = [ - google_pubsub_subscription_iam_binding.default + google_pubsub_subscription_iam_binding.authoritative, + google_pubsub_subscription_iam_binding.bindings ] } @@ -55,6 +58,7 @@ output "topic" { description = "Topic resource." value = google_pubsub_topic.default depends_on = [ - google_pubsub_topic_iam_binding.default + google_pubsub_topic_iam_binding.authoritative, + google_pubsub_topic_iam_binding.bindings ] } diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index 35dfc172..370c42fa 100644 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,63 +14,41 @@ * limitations under the License. */ -variable "bigquery_subscription_configs" { - description = "Configuration parameters for BigQuery subscriptions." - type = map(object({ - table = string - use_topic_schema = bool - write_metadata = bool - drop_unknown_fields = bool - })) - default = {} -} - -variable "cloud_storage_subscription_configs" { - description = "Configuration parameters for Cloud Storage subscriptions." - type = map(object({ - bucket = string - filename_prefix = string - filename_suffix = string - max_duration = string - max_bytes = number - avro_config = object({ - write_metadata = bool - }) - })) - default = {} -} - -variable "dead_letter_configs" { - description = "Per-subscription dead letter policy configuration." - type = map(object({ - topic = string - max_delivery_attempts = number - })) - default = {} -} - -variable "defaults" { - description = "Subscription defaults for options." - type = object({ - ack_deadline_seconds = number - message_retention_duration = string - retain_acked_messages = bool - expiration_policy_ttl = string - filter = string - }) - default = { - ack_deadline_seconds = null - message_retention_duration = null - retain_acked_messages = null - expiration_policy_ttl = null - filter = null - } -} - variable "iam" { description = "IAM bindings for topic in {ROLE => [MEMBERS]} format." type = map(list(string)) default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Keyring individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } variable "kms_key" { @@ -83,6 +61,7 @@ variable "labels" { description = "Labels." type = map(string) default = {} + nullable = false } variable "message_retention_duration" { @@ -101,23 +80,11 @@ variable "project_id" { type = string } -variable "push_configs" { - description = "Push subscription configurations." - type = map(object({ - attributes = map(string) - endpoint = string - oidc_token = object({ - audience = string - service_account_email = string - }) - })) - default = {} -} - variable "regions" { description = "List of regions used to set persistence policy." type = list(string) default = [] + nullable = false } variable "schema" { @@ -130,23 +97,72 @@ variable "schema" { default = null } -variable "subscription_iam" { - description = "IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format." - type = map(map(list(string))) - default = {} -} - variable "subscriptions" { description = "Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null." type = map(object({ - labels = map(string) - options = object({ - ack_deadline_seconds = number - message_retention_duration = string - retain_acked_messages = bool - expiration_policy_ttl = string - filter = string - }) + labels = optional(map(string)) + ack_deadline_seconds = optional(number) + message_retention_duration = optional(string) + retain_acked_messages = optional(bool, false) + expiration_policy_ttl = optional(string) + filter = optional(string) + enable_message_ordering = optional(bool, false) + enable_exactly_once_delivery = optional(bool, false) + dead_letter_policy = optional(object({ + topic = string + max_delivery_attempts = optional(number) + })) + retry_policy = optional(object({ + minimum_backoff = optional(number) + maximum_backoff = optional(number) + })) + + bigquery = optional(object({ + table = string + use_topic_schema = optional(bool, false) + write_metadata = optional(bool, false) + drop_unknown_fields = optional(bool, false) + })) + cloud_storage = optional(object({ + bucket = string + filename_prefix = optional(string) + filename_suffix = optional(string) + max_duration = optional(string) + max_bytes = optional(number) + avro_config = optional(object({ + write_metadata = optional(bool, false) + })) + })) + push = optional(object({ + endpoint = string + attributes = optional(map(string)) + no_wrapper = optional(bool, false) + oidc_token = optional(object({ + audience = optional(string) + service_account_email = string + })) + })) + + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) })) - default = {} + default = {} + nullable = false } diff --git a/tests/modules/pubsub/examples/simple.yaml b/tests/modules/pubsub/examples/simple.yaml index 51094a51..6fe54ec6 100644 --- a/tests/modules/pubsub/examples/simple.yaml +++ b/tests/modules/pubsub/examples/simple.yaml @@ -16,14 +16,14 @@ values: module.pubsub.google_pubsub_topic.default: name: my-topic project: my-project - module.pubsub.google_pubsub_topic_iam_binding.default["roles/pubsub.subscriber"]: + module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.subscriber"]: condition: [] members: - user:user1@example.com project: my-project role: roles/pubsub.subscriber topic: my-topic - module.pubsub.google_pubsub_topic_iam_binding.default["roles/pubsub.viewer"]: + module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.viewer"]: condition: [] members: - group:foo@example.com diff --git a/tests/modules/pubsub/examples/subscription-iam.yaml b/tests/modules/pubsub/examples/subscription-iam.yaml index d0fa9fb6..42ed2565 100644 --- a/tests/modules/pubsub/examples/subscription-iam.yaml +++ b/tests/modules/pubsub/examples/subscription-iam.yaml @@ -13,10 +13,10 @@ # limitations under the License. values: - module.pubsub.google_pubsub_subscription_iam_binding.default["test-1.roles/pubsub.subscriber"]: + module.pubsub.google_pubsub_subscription_iam_binding.authoritative["test-1.roles/pubsub.subscriber"]: condition: [] members: - - user:user1@ludomagno.net + - user:user1@example.com project: my-project role: roles/pubsub.subscriber subscription: test-1 diff --git a/tests/modules/pubsub/examples/subscriptions.yaml b/tests/modules/pubsub/examples/subscriptions.yaml index a87a6d47..b1a94212 100644 --- a/tests/modules/pubsub/examples/subscriptions.yaml +++ b/tests/modules/pubsub/examples/subscriptions.yaml @@ -16,22 +16,22 @@ values: module.pubsub.google_pubsub_subscription.default["test-pull"]: bigquery_config: [] dead_letter_policy: [] - enable_exactly_once_delivery: null - enable_message_ordering: null + enable_exactly_once_delivery: False + enable_message_ordering: False filter: null labels: null message_retention_duration: 604800s name: test-pull project: my-project push_config: [] - retain_acked_messages: null + retain_acked_messages: False retry_policy: [] topic: my-topic module.pubsub.google_pubsub_subscription.default["test-pull-override"]: bigquery_config: [] dead_letter_policy: [] - enable_exactly_once_delivery: null - enable_message_ordering: null + enable_exactly_once_delivery: False + enable_message_ordering: False filter: null labels: test: override @@ -39,7 +39,7 @@ values: name: test-pull-override project: my-project push_config: [] - retain_acked_messages: true + retain_acked_messages: True retry_policy: [] topic: my-topic module.pubsub.google_pubsub_topic.default: