From 54e9738c393b66a481b04d47ae5ab297a3dabb46 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Sun, 20 Nov 2022 13:26:33 +0100 Subject: [PATCH] Add schemas to Pubsub topic module. Pubsub topics can now have schemas (https://cloud.google.com/pubsub/docs/admin#schemas). This PR adds an option to set the schema settings and create a new optional resource of type `google_pubsub_schema` attached to the `google_pubsub_topic`. --- modules/pubsub/README.md | 37 ++++++++++++++++++++++++++++++++++++- modules/pubsub/main.tf | 17 +++++++++++++++++ modules/pubsub/outputs.tf | 16 ++++++++++++++++ modules/pubsub/variables.tf | 10 ++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index b75aaf6d..d20d1cc4 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -1,6 +1,6 @@ # Google Cloud Pub/Sub Module -This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels. +This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels, as well as schemas. ## Examples @@ -20,6 +20,38 @@ module "pubsub" { # tftest modules=1 resources=3 ``` +### Topic with schema + +```hcl +module "topic_with_schema" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + schema = { + msg_encoding = "JSON" + schema_type = "AVRO" + definition = jsonencode({ + "type" = "record", + "name" = "Avro", + "fields" : [{ + "name" = "StringField", + "type" = "string" + }, + { + "name" = "FloatField", + "type" = "float" + }, + { + "name" = "BooleanField", + "type" = "boolean" + }, + ] + }) + } +} +# tftest modules=1 resources=3 +``` + ### Subscriptions Subscriptions are defined with the `subscriptions` variable, allowing optional configuration of per-subscription defaults. Push subscriptions need extra configuration, shown in the following example. @@ -104,6 +136,7 @@ module "pubsub" { | [message_retention_duration](variables.tf#L62) | Minimum duration to retain a message after it is published to the topic. | string | | null | | [push_configs](variables.tf#L78) | Push subscription configurations. | map(object({…})) | | {} | | [regions](variables.tf#L91) | List of regions used to set persistence policy. | list(string) | | [] | +| [schema](variables.tf#L118) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | | [subscription_iam](variables.tf#L97) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | | [subscriptions](variables.tf#L103) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | @@ -112,6 +145,8 @@ module "pubsub" { | name | description | sensitive | |---|---|:---:| | [id](outputs.tf#L17) | Topic id. | | +| [schema](outputs.tf#L59) | Schema resource. | | +| [schema_id](outputs.tf#L51) | Schema resource id. | | | [subscription_id](outputs.tf#L25) | Subscription ids. | | | [subscriptions](outputs.tf#L35) | Subscription resources. | | | [topic](outputs.tf#L43) | Topic resource. | | diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index 2645de02..b0beb8b7 100644 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -35,6 +35,14 @@ locals { } } +resource "google_pubsub_schema" "default" { + count = var.schema == null ? 0 : 1 + name = format("%s-%s", var.name, "schema") + type = var.schema.schema_type + definition = var.schema.definition + project = var.project_id +} + resource "google_pubsub_topic" "default" { project = var.project_id name = var.name @@ -48,6 +56,15 @@ resource "google_pubsub_topic" "default" { allowed_persistence_regions = var.regions } } + + depends_on = [google_pubsub_schema.default] + dynamic "schema_settings" { + for_each = var.schema == null ? [] : [""] + content { + schema = google_pubsub_schema.default[0].id + encoding = var.schema.msg_encoding + } + } } resource "google_pubsub_topic_iam_binding" "default" { diff --git a/modules/pubsub/outputs.tf b/modules/pubsub/outputs.tf index c26eb4d9..971a1f34 100644 --- a/modules/pubsub/outputs.tf +++ b/modules/pubsub/outputs.tf @@ -47,3 +47,19 @@ output "topic" { google_pubsub_topic_iam_binding.default ] } + +output "schema_id" { + description = "Schema resource id." + value = google_pubsub_schema.default[0].id + depends_on = [ + google_pubsub_schema.default + ] +} + +output "schema" { + description = "Schema resource." + value = google_pubsub_schema.default[0] + depends_on = [ + google_pubsub_schema.default + ] +} diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index 009a12fb..503ff690 100644 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -114,3 +114,13 @@ variable "subscriptions" { })) default = {} } + +variable "schema" { + description = "Topic schema. If set, all messages in this topic should follow this schema." + type = object({ + schema_type = string + definition = string + msg_encoding = optional(string, "ENCODING_UNSPECIFIED") + }) + default = null +}