Update IAM for pubsub topics and subscriptions

This commit is contained in:
Julio Castillo 2023-09-16 23:52:07 +02:00
parent 45203fe86c
commit 9d61c6e26d
8 changed files with 317 additions and 212 deletions

View File

@ -61,16 +61,10 @@ module "pubsub" {
project_id = "my-project"
name = "my-topic"
subscriptions = {
test-pull = null
test-pull = {}
test-pull-override = {
labels = { test = "override" }
options = {
ack_deadline_seconds = null
message_retention_duration = null
retain_acked_messages = true
expiration_policy_ttl = null
filter = null
}
labels = { test = "override" }
retain_acked_messages = true
}
}
}
@ -87,13 +81,10 @@ module "pubsub" {
project_id = "my-project"
name = "my-topic"
subscriptions = {
test-push = null
}
push_configs = {
test-push = {
endpoint = "https://example.com/foo"
attributes = null
oidc_token = null
push = {
endpoint = "https://example.com/foo"
}
}
}
}
@ -110,14 +101,13 @@ module "pubsub" {
project_id = "my-project"
name = "my-topic"
subscriptions = {
test-bigquery = null
}
bigquery_subscription_configs = {
test-bigquery = {
table = "my_project_id:my_dataset.my_table"
use_topic_schema = true
write_metadata = false
drop_unknown_fields = true
bigquery = {
table = "my_project_id:my_dataset.my_table"
use_topic_schema = true
write_metadata = false
drop_unknown_fields = true
}
}
}
}
@ -134,17 +124,16 @@ module "pubsub" {
project_id = "my-project"
name = "my-topic"
subscriptions = {
test-cloudstorage = null
}
cloud_storage_subscription_configs = {
test-cloudstorage = {
bucket = "my-bucket"
filename_prefix = "test_prefix"
filename_suffix = "test_suffix"
max_duration = "100s"
max_bytes = 1000
avro_config = {
write_metadata = true
cloud_storage = {
bucket = "my-bucket"
filename_prefix = "test_prefix"
filename_suffix = "test_suffix"
max_duration = "100s"
max_bytes = 1000
avro_config = {
write_metadata = true
}
}
}
}
@ -159,12 +148,10 @@ module "pubsub" {
project_id = "my-project"
name = "my-topic"
subscriptions = {
test-1 = null
test-1 = null
}
subscription_iam = {
test-1 = {
"roles/pubsub.subscriber" = ["user:user1@ludomagno.net"]
iam = {
"roles/pubsub.subscriber" = ["user:user1@example.com"]
}
}
}
}

140
modules/pubsub/iam.tf Normal file
View File

@ -0,0 +1,140 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the authoritative.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
subscription_iam = flatten([
for k, v in var.subscriptions : [
for role, members in v.iam : {
subscription = k
role = role
members = members
}
]
])
subscription_iam_bindings = merge([
for k, v in var.subscriptions : {
for binding_key, data in v.iam_bindings :
binding_key => {
subscription = k
role = data.role
members = data.members
condition = data.condition
}
}
]...)
subscription_iam_bindings_additive = merge([
for k, v in var.subscriptions : {
for binding_key, data in v.iam_bindings_additive :
binding_key => {
subscription = k
role = data.role
member = data.member
condition = data.condition
}
}
]...)
}
moved {
from = google_pubsub_topic_iam_binding.default
to = google_pubsub_topic_iam_binding.authoritative
}
resource "google_pubsub_topic_iam_binding" "authoritative" {
for_each = var.iam
project = var.project_id
topic = google_pubsub_topic.default.name
role = each.key
members = each.value
}
resource "google_pubsub_topic_iam_binding" "bindings" {
for_each = var.iam_bindings
topic = google_pubsub_topic.default.name
role = each.value.role
members = each.value.members
dynamic "condition" {
for_each = each.value.condition == null ? [] : [""]
content {
expression = each.value.condition.expression
title = each.value.condition.title
description = each.value.condition.description
}
}
}
resource "google_pubsub_topic_iam_member" "bindings" {
for_each = var.iam_bindings_additive
topic = google_pubsub_topic.default.name
role = each.value.role
member = each.value.member
dynamic "condition" {
for_each = each.value.condition == null ? [] : [""]
content {
expression = each.value.condition.expression
title = each.value.condition.title
description = each.value.condition.description
}
}
}
moved {
from = google_pubsub_subscription_iam_binding.default
to = google_pubsub_subscription_iam_binding.authoritative
}
resource "google_pubsub_subscription_iam_binding" "authoritative" {
for_each = {
for binding in local.subscription_iam :
"${binding.subscription}.${binding.role}" => binding
}
project = var.project_id
subscription = google_pubsub_subscription.default[each.value.subscription].name
role = each.value.role
members = each.value.members
}
resource "google_pubsub_subscription_iam_binding" "bindings" {
for_each = local.subscription_iam_bindings
project = var.project_id
subscription = google_pubsub_subscription.default[each.value.subscription].name
role = each.value.role
members = each.value.members
dynamic "condition" {
for_each = each.value.condition == null ? [] : [""]
content {
expression = each.value.condition.expression
title = each.value.condition.title
description = each.value.condition.description
}
}
}
resource "google_pubsub_subscription_iam_member" "members" {
for_each = local.subscription_iam_bindings_additive
project = var.project_id
subscription = google_pubsub_subscription.default[each.value.subscription].name
role = each.value.role
member = each.value.member
dynamic "condition" {
for_each = each.value.condition == null ? [] : [""]
content {
expression = each.value.condition.expression
title = each.value.condition.title
description = each.value.condition.description
}
}
}

View File

@ -1,5 +1,5 @@
/**
* Copyright 2022 Google LLC
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,27 +15,6 @@
*/
locals {
sub_iam_members = flatten([
for sub, roles in var.subscription_iam : [
for role, members in roles : {
sub = sub
role = role
members = members
}
]
])
metadata_config = {
for k, v in var.cloud_storage_subscription_configs : k => v.avro_config
}
oidc_config = {
for k, v in var.push_configs : k => v.oidc_token
}
subscriptions = {
for k, v in var.subscriptions : k => {
labels = try(v.labels, v, null) == null ? var.labels : v.labels
options = try(v.options, v, null) == null ? var.defaults : v.options
}
}
topic_id_static = "projects/${var.project_id}/topics/${var.name}"
}
@ -70,94 +49,73 @@ resource "google_pubsub_topic" "default" {
}
}
resource "google_pubsub_topic_iam_binding" "default" {
for_each = var.iam
project = var.project_id
topic = google_pubsub_topic.default.name
role = each.key
members = each.value
}
resource "google_pubsub_subscription" "default" {
for_each = local.subscriptions
project = var.project_id
name = each.key
topic = google_pubsub_topic.default.name
labels = each.value.labels
ack_deadline_seconds = each.value.options.ack_deadline_seconds
message_retention_duration = each.value.options.message_retention_duration
retain_acked_messages = each.value.options.retain_acked_messages
filter = each.value.options.filter
for_each = var.subscriptions
project = var.project_id
name = each.key
topic = google_pubsub_topic.default.name
labels = each.value.labels
ack_deadline_seconds = each.value.ack_deadline_seconds
message_retention_duration = each.value.message_retention_duration
retain_acked_messages = each.value.retain_acked_messages
filter = each.value.filter
enable_message_ordering = each.value.enable_message_ordering
enable_exactly_once_delivery = each.value.enable_exactly_once_delivery
dynamic "expiration_policy" {
for_each = each.value.options.expiration_policy_ttl == null ? [] : [""]
for_each = each.value.expiration_policy_ttl == null ? [] : [""]
content {
ttl = each.value.options.expiration_policy_ttl
ttl = each.value.expiration_policy_ttl
}
}
dynamic "dead_letter_policy" {
for_each = try(var.dead_letter_configs[each.key], null) == null ? [] : [""]
for_each = each.value.dead_letter_policy == null ? [] : [""]
content {
dead_letter_topic = var.dead_letter_configs[each.key].topic
max_delivery_attempts = var.dead_letter_configs[each.key].max_delivery_attempts
dead_letter_topic = each.value.dead_letter_policy.topic
max_delivery_attempts = each.value.dead_letter_policy.max_delivery_attempts
}
}
dynamic "push_config" {
for_each = try(var.push_configs[each.key], null) == null ? [] : [""]
for_each = each.value.push == null ? [] : [""]
content {
push_endpoint = var.push_configs[each.key].endpoint
attributes = var.push_configs[each.key].attributes
push_endpoint = each.value.push.endpoint
attributes = each.value.push.attributes
dynamic "oidc_token" {
for_each = (
local.oidc_config[each.key] == null ? [] : [""]
)
for_each = each.value.push.oidc_token == null ? [] : [""]
content {
service_account_email = local.oidc_config[each.key].service_account_email
audience = local.oidc_config[each.key].audience
service_account_email = each.value.push.oidc_token.service_account_email
audience = each.value.push.oidc_token.audience
}
}
}
}
dynamic "bigquery_config" {
for_each = try(var.bigquery_subscription_configs[each.key], null) == null ? [] : [""]
for_each = each.value.bigquery == null ? [] : [""]
content {
table = var.bigquery_subscription_configs[each.key].table
use_topic_schema = var.bigquery_subscription_configs[each.key].use_topic_schema
write_metadata = var.bigquery_subscription_configs[each.key].write_metadata
drop_unknown_fields = var.bigquery_subscription_configs[each.key].drop_unknown_fields
table = each.value.bigquery.table
use_topic_schema = each.value.bigquery.use_topic_schema
write_metadata = each.value.bigquery.write_metadata
drop_unknown_fields = each.value.bigquery.drop_unknown_fields
}
}
dynamic "cloud_storage_config" {
for_each = try(var.cloud_storage_subscription_configs[each.key], null) == null ? [] : [""]
for_each = each.value.cloud_storage == null ? [] : [""]
content {
bucket = var.cloud_storage_subscription_configs[each.key].bucket
filename_prefix = var.cloud_storage_subscription_configs[each.key].filename_prefix
filename_suffix = var.cloud_storage_subscription_configs[each.key].filename_suffix
max_duration = var.cloud_storage_subscription_configs[each.key].max_duration
max_bytes = var.cloud_storage_subscription_configs[each.key].max_bytes
bucket = each.value.cloud_storage.bucket
filename_prefix = each.value.cloud_storage.filename_prefix
filename_suffix = each.value.cloud_storage.filename_suffix
max_duration = each.value.cloud_storage.max_duration
max_bytes = each.value.cloud_storage.max_bytes
dynamic "avro_config" {
for_each = (
local.metadata_config[each.key] == null ? [] : [""]
)
for_each = each.value.cloud_storage.avro_config == null ? [] : [""]
content {
write_metadata = local.metadata_config[each.key].write_metadata
write_metadata = each.value.cloud_storage.avro_config.write_metadata
}
}
}
}
}
resource "google_pubsub_subscription_iam_binding" "default" {
for_each = {
for binding in local.sub_iam_members :
"${binding.sub}.${binding.role}" => binding
}
project = var.project_id
subscription = google_pubsub_subscription.default[each.value.sub].name
role = each.value.role
members = each.value.members
}

View File

@ -1,5 +1,5 @@
/**
* Copyright 2022 Google LLC
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,7 +19,8 @@ output "id" {
value = local.topic_id_static
depends_on = [
google_pubsub_topic.default,
google_pubsub_topic_iam_binding.default
google_pubsub_topic_iam_binding.authoritative,
google_pubsub_topic_iam_binding.bindings
]
}
@ -39,7 +40,8 @@ output "subscription_id" {
for k, v in google_pubsub_subscription.default : k => v.id
}
depends_on = [
google_pubsub_subscription_iam_binding.default
google_pubsub_subscription_iam_binding.authoritative,
google_pubsub_subscription_iam_binding.bindings
]
}
@ -47,7 +49,8 @@ output "subscriptions" {
description = "Subscription resources."
value = google_pubsub_subscription.default
depends_on = [
google_pubsub_subscription_iam_binding.default
google_pubsub_subscription_iam_binding.authoritative,
google_pubsub_subscription_iam_binding.bindings
]
}
@ -55,6 +58,7 @@ output "topic" {
description = "Topic resource."
value = google_pubsub_topic.default
depends_on = [
google_pubsub_topic_iam_binding.default
google_pubsub_topic_iam_binding.authoritative,
google_pubsub_topic_iam_binding.bindings
]
}

View File

@ -1,5 +1,5 @@
/**
* Copyright 2022 Google LLC
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,63 +14,41 @@
* limitations under the License.
*/
variable "bigquery_subscription_configs" {
description = "Configuration parameters for BigQuery subscriptions."
type = map(object({
table = string
use_topic_schema = bool
write_metadata = bool
drop_unknown_fields = bool
}))
default = {}
}
variable "cloud_storage_subscription_configs" {
description = "Configuration parameters for Cloud Storage subscriptions."
type = map(object({
bucket = string
filename_prefix = string
filename_suffix = string
max_duration = string
max_bytes = number
avro_config = object({
write_metadata = bool
})
}))
default = {}
}
variable "dead_letter_configs" {
description = "Per-subscription dead letter policy configuration."
type = map(object({
topic = string
max_delivery_attempts = number
}))
default = {}
}
variable "defaults" {
description = "Subscription defaults for options."
type = object({
ack_deadline_seconds = number
message_retention_duration = string
retain_acked_messages = bool
expiration_policy_ttl = string
filter = string
})
default = {
ack_deadline_seconds = null
message_retention_duration = null
retain_acked_messages = null
expiration_policy_ttl = null
filter = null
}
}
variable "iam" {
description = "IAM bindings for topic in {ROLE => [MEMBERS]} format."
type = map(list(string))
default = {}
nullable = false
}
variable "iam_bindings" {
description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary."
type = map(object({
members = list(string)
role = string
condition = optional(object({
expression = string
title = string
description = optional(string)
}))
}))
nullable = false
default = {}
}
variable "iam_bindings_additive" {
description = "Keyring individual additive IAM bindings. Keys are arbitrary."
type = map(object({
member = string
role = string
condition = optional(object({
expression = string
title = string
description = optional(string)
}))
}))
nullable = false
default = {}
}
variable "kms_key" {
@ -83,6 +61,7 @@ variable "labels" {
description = "Labels."
type = map(string)
default = {}
nullable = false
}
variable "message_retention_duration" {
@ -101,23 +80,11 @@ variable "project_id" {
type = string
}
variable "push_configs" {
description = "Push subscription configurations."
type = map(object({
attributes = map(string)
endpoint = string
oidc_token = object({
audience = string
service_account_email = string
})
}))
default = {}
}
variable "regions" {
description = "List of regions used to set persistence policy."
type = list(string)
default = []
nullable = false
}
variable "schema" {
@ -130,23 +97,72 @@ variable "schema" {
default = null
}
variable "subscription_iam" {
description = "IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format."
type = map(map(list(string)))
default = {}
}
variable "subscriptions" {
description = "Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null."
type = map(object({
labels = map(string)
options = object({
ack_deadline_seconds = number
message_retention_duration = string
retain_acked_messages = bool
expiration_policy_ttl = string
filter = string
})
labels = optional(map(string))
ack_deadline_seconds = optional(number)
message_retention_duration = optional(string)
retain_acked_messages = optional(bool, false)
expiration_policy_ttl = optional(string)
filter = optional(string)
enable_message_ordering = optional(bool, false)
enable_exactly_once_delivery = optional(bool, false)
dead_letter_policy = optional(object({
topic = string
max_delivery_attempts = optional(number)
}))
retry_policy = optional(object({
minimum_backoff = optional(number)
maximum_backoff = optional(number)
}))
bigquery = optional(object({
table = string
use_topic_schema = optional(bool, false)
write_metadata = optional(bool, false)
drop_unknown_fields = optional(bool, false)
}))
cloud_storage = optional(object({
bucket = string
filename_prefix = optional(string)
filename_suffix = optional(string)
max_duration = optional(string)
max_bytes = optional(number)
avro_config = optional(object({
write_metadata = optional(bool, false)
}))
}))
push = optional(object({
endpoint = string
attributes = optional(map(string))
no_wrapper = optional(bool, false)
oidc_token = optional(object({
audience = optional(string)
service_account_email = string
}))
}))
iam = optional(map(list(string)), {})
iam_bindings = optional(map(object({
members = list(string)
role = string
condition = optional(object({
expression = string
title = string
description = optional(string)
}))
})), {})
iam_bindings_additive = optional(map(object({
member = string
role = string
condition = optional(object({
expression = string
title = string
description = optional(string)
}))
})), {})
}))
default = {}
default = {}
nullable = false
}

View File

@ -16,14 +16,14 @@ values:
module.pubsub.google_pubsub_topic.default:
name: my-topic
project: my-project
module.pubsub.google_pubsub_topic_iam_binding.default["roles/pubsub.subscriber"]:
module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.subscriber"]:
condition: []
members:
- user:user1@example.com
project: my-project
role: roles/pubsub.subscriber
topic: my-topic
module.pubsub.google_pubsub_topic_iam_binding.default["roles/pubsub.viewer"]:
module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.viewer"]:
condition: []
members:
- group:foo@example.com

View File

@ -13,10 +13,10 @@
# limitations under the License.
values:
module.pubsub.google_pubsub_subscription_iam_binding.default["test-1.roles/pubsub.subscriber"]:
module.pubsub.google_pubsub_subscription_iam_binding.authoritative["test-1.roles/pubsub.subscriber"]:
condition: []
members:
- user:user1@ludomagno.net
- user:user1@example.com
project: my-project
role: roles/pubsub.subscriber
subscription: test-1

View File

@ -16,22 +16,22 @@ values:
module.pubsub.google_pubsub_subscription.default["test-pull"]:
bigquery_config: []
dead_letter_policy: []
enable_exactly_once_delivery: null
enable_message_ordering: null
enable_exactly_once_delivery: False
enable_message_ordering: False
filter: null
labels: null
message_retention_duration: 604800s
name: test-pull
project: my-project
push_config: []
retain_acked_messages: null
retain_acked_messages: False
retry_policy: []
topic: my-topic
module.pubsub.google_pubsub_subscription.default["test-pull-override"]:
bigquery_config: []
dead_letter_policy: []
enable_exactly_once_delivery: null
enable_message_ordering: null
enable_exactly_once_delivery: False
enable_message_ordering: False
filter: null
labels:
test: override
@ -39,7 +39,7 @@ values:
name: test-pull-override
project: my-project
push_config: []
retain_acked_messages: true
retain_acked_messages: True
retry_policy: []
topic: my-topic
module.pubsub.google_pubsub_topic.default: