From 688c6cfb79f8d0e8ee231be9fe870e70b38f550d Mon Sep 17 00:00:00 2001 From: andybubu <147597622+andybubu@users.noreply.github.com> Date: Wed, 20 Mar 2024 17:30:30 +0100 Subject: [PATCH] feat: add e2e test for pubsub module (#2163) * feat: add e2e test for pubsub module --- modules/pubsub/README.md | 45 +++++++++------- tests/fixtures/bigquery-dataset.tf | 38 +++++++++++++ tests/fixtures/gcs.tf | 23 ++++++++ .../examples/bigquery-subscription.yaml | 13 +++-- .../examples/cloud-storage-subscription.yaml | 53 +++++++++++++++++++ .../pubsub/examples/push-subscription.yaml | 25 +++++++-- tests/modules/pubsub/examples/schema.yaml | 15 ++++-- tests/modules/pubsub/examples/simple.yaml | 16 ++++-- .../pubsub/examples/subscription-iam.yaml | 30 ++++++++++- .../pubsub/examples/subscriptions.yaml | 34 ++++++++---- 10 files changed, 245 insertions(+), 47 deletions(-) create mode 100644 tests/fixtures/bigquery-dataset.tf create mode 100644 tests/fixtures/gcs.tf create mode 100644 tests/modules/pubsub/examples/cloud-storage-subscription.yaml diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 69a18dbe..6d924d90 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -9,14 +9,14 @@ This module allows managing a single Pub/Sub topic, including multiple subscript ```hcl module "pubsub" { source = "./fabric/modules/pubsub" - project_id = "my-project" + project_id = var.project_id name = "my-topic" iam = { - "roles/pubsub.viewer" = ["group:foo@example.com"] - "roles/pubsub.subscriber" = ["user:user1@example.com"] + "roles/pubsub.viewer" = ["group:${var.group_email}"] + "roles/pubsub.subscriber" = ["serviceAccount:${var.service_account.email}"] } } -# tftest modules=1 resources=3 inventory=simple.yaml +# tftest modules=1 resources=3 inventory=simple.yaml e2e ``` ### Topic with schema @@ -24,7 +24,7 @@ module "pubsub" { ```hcl module "topic_with_schema" { source = "./fabric/modules/pubsub" - project_id = "my-project" + project_id = var.project_id name = "my-topic" schema = { msg_encoding = "JSON" @@ -48,7 +48,7 @@ module "topic_with_schema" { }) } } -# tftest modules=1 resources=2 inventory=schema.yaml +# tftest modules=1 resources=2 inventory=schema.yaml e2e ``` ### Subscriptions @@ -58,7 +58,7 @@ Subscriptions are defined with the `subscriptions` variable, allowing optional c ```hcl module "pubsub" { source = "./fabric/modules/pubsub" - project_id = "my-project" + project_id = var.project_id name = "my-topic" subscriptions = { test-pull = {} @@ -68,7 +68,7 @@ module "pubsub" { } } } -# tftest modules=1 resources=3 inventory=subscriptions.yaml +# tftest modules=1 resources=3 inventory=subscriptions.yaml e2e ``` ### Push subscriptions @@ -78,7 +78,7 @@ Push subscriptions need extra configuration in the `push_configs` variable. ```hcl module "pubsub" { source = "./fabric/modules/pubsub" - project_id = "my-project" + project_id = var.project_id name = "my-topic" subscriptions = { test-push = { @@ -88,7 +88,7 @@ module "pubsub" { } } } -# tftest modules=1 resources=2 +# tftest modules=1 resources=2 inventory=push-subscription.yaml e2e ``` ### BigQuery subscriptions @@ -98,12 +98,12 @@ BigQuery subscriptions need extra configuration in the `bigquery_subscription_co ```hcl module "pubsub" { source = "./fabric/modules/pubsub" - project_id = "my-project" + project_id = var.project_id name = "my-topic" subscriptions = { test-bigquery = { bigquery = { - table = "my_project_id:my_dataset.my_table" + table = "${module.bigquery-dataset.tables["my_table"].project}:${module.bigquery-dataset.tables["my_table"].dataset_id}.${module.bigquery-dataset.tables["my_table"].table_id}" use_topic_schema = true write_metadata = false drop_unknown_fields = true @@ -111,7 +111,7 @@ module "pubsub" { } } } -# tftest modules=1 resources=2 +# tftest modules=2 resources=5 fixtures=fixtures/bigquery-dataset.tf inventory=bigquery-subscription.yaml e2e ``` ### Cloud Storage subscriptions @@ -121,13 +121,13 @@ Cloud Storage subscriptions need extra configuration in the `cloud_storage_subsc ```hcl module "pubsub" { source = "./fabric/modules/pubsub" - project_id = "my-project" + project_id = var.project_id name = "my-topic" subscriptions = { test-cloudstorage = { cloud_storage = { - bucket = "my-bucket" - filename_prefix = "test_prefix" + bucket = module.gcs.name + filename_prefix = var.prefix filename_suffix = "test_suffix" max_duration = "100s" max_bytes = 1000 @@ -138,24 +138,24 @@ module "pubsub" { } } } -# tftest modules=1 resources=2 +# tftest modules=2 resources=4 fixtures=fixtures/gcs.tf inventory=cloud-storage-subscription.yaml e2e ``` ### Subscriptions with IAM ```hcl module "pubsub" { source = "./fabric/modules/pubsub" - project_id = "my-project" + project_id = var.project_id name = "my-topic" subscriptions = { test-1 = { iam = { - "roles/pubsub.subscriber" = ["user:user1@example.com"] + "roles/pubsub.subscriber" = ["serviceAccount:${var.service_account.email}"] } } } } -# tftest modules=1 resources=3 +# tftest modules=1 resources=3 inventory=subscription-iam.yaml e2e ``` ## Variables @@ -184,4 +184,9 @@ module "pubsub" { | [subscription_id](outputs.tf#L37) | Subscription ids. | | | [subscriptions](outputs.tf#L48) | Subscription resources. | | | [topic](outputs.tf#L57) | Topic resource. | | + +## Fixtures + +- [bigquery-dataset.tf](../../tests/fixtures/bigquery-dataset.tf) +- [gcs.tf](../../tests/fixtures/gcs.tf) diff --git a/tests/fixtures/bigquery-dataset.tf b/tests/fixtures/bigquery-dataset.tf new file mode 100644 index 00000000..bd47aa92 --- /dev/null +++ b/tests/fixtures/bigquery-dataset.tf @@ -0,0 +1,38 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + schema = jsonencode([ + { name = "data", type = "STRING" }, + { name = "freq", type = "INT64" } + ]) +} + +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = var.project_id + id = "my_dataset" + tables = { + my_table = { + deletion_protection = false + schema = local.schema + partitioning = { + time = { type = "DAY", expiration_ms = null } + } + } + } + iam = { + "roles/bigquery.dataEditor" = ["serviceAccount:service-${var.project_number}@gcp-sa-pubsub.iam.gserviceaccount.com"] + } +} \ No newline at end of file diff --git a/tests/fixtures/gcs.tf b/tests/fixtures/gcs.tf new file mode 100644 index 00000000..107a36e4 --- /dev/null +++ b/tests/fixtures/gcs.tf @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "gcs" { + source = "./fabric/modules/gcs" + project_id = var.project_id + prefix = var.prefix + name = "my-bucket" + iam = { + "roles/storage.admin" = ["serviceAccount:service-${var.project_number}@gcp-sa-pubsub.iam.gserviceaccount.com"] + } +} \ No newline at end of file diff --git a/tests/modules/pubsub/examples/bigquery-subscription.yaml b/tests/modules/pubsub/examples/bigquery-subscription.yaml index c0c20e74..2b8d154f 100644 --- a/tests/modules/pubsub/examples/bigquery-subscription.yaml +++ b/tests/modules/pubsub/examples/bigquery-subscription.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,16 +16,21 @@ values: module.pubsub.google_pubsub_subscription.default["test-bigquery"]: bigquery_config: - drop_unknown_fields: true - table: my_project_id:my_dataset.my_table + table: project-id:my_dataset.my_table use_topic_schema: true write_metadata: false name: test-bigquery - project: my-project + project: project-id topic: my-topic module.pubsub.google_pubsub_topic.default: name: my-topic - project: my-project + project: project-id counts: + google_bigquery_dataset: 1 + google_bigquery_dataset_iam_binding: 1 + google_bigquery_table: 1 google_pubsub_subscription: 1 google_pubsub_topic: 1 + modules: 2 + resources: 5 diff --git a/tests/modules/pubsub/examples/cloud-storage-subscription.yaml b/tests/modules/pubsub/examples/cloud-storage-subscription.yaml new file mode 100644 index 00000000..b5493c39 --- /dev/null +++ b/tests/modules/pubsub/examples/cloud-storage-subscription.yaml @@ -0,0 +1,53 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.pubsub.google_pubsub_subscription.default["test-cloudstorage"]: + bigquery_config: [] + cloud_storage_config: + - avro_config: + - write_metadata: true + bucket: test-my-bucket + filename_prefix: test + filename_suffix: test_suffix + max_bytes: 1000 + max_duration: 100s + dead_letter_policy: [] + enable_exactly_once_delivery: false + enable_message_ordering: false + filter: null + labels: null + message_retention_duration: 604800s + name: test-cloudstorage + project: project-id + push_config: [] + retain_acked_messages: false + retry_policy: [] + timeouts: null + topic: my-topic + module.pubsub.google_pubsub_topic.default: + kms_key_name: null + labels: null + message_retention_duration: null + name: my-topic + project: project-id + timeouts: null + +counts: + google_pubsub_subscription: 1 + google_pubsub_topic: 1 + google_storage_bucket: 1 + google_storage_bucket_iam_binding: 1 + modules: 2 + resources: 4 diff --git a/tests/modules/pubsub/examples/push-subscription.yaml b/tests/modules/pubsub/examples/push-subscription.yaml index efa3cc1b..15d1c11a 100644 --- a/tests/modules/pubsub/examples/push-subscription.yaml +++ b/tests/modules/pubsub/examples/push-subscription.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,19 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. + values: module.pubsub.google_pubsub_subscription.default["test-push"]: + bigquery_config: [] + cloud_storage_config: [] + dead_letter_policy: [] + enable_exactly_once_delivery: false + enable_message_ordering: false + filter: null + labels: null + message_retention_duration: 604800s name: test-push - project: my-project + project: project-id push_config: - attributes: null + no_wrapper: [] oidc_token: [] push_endpoint: https://example.com/foo + retain_acked_messages: false + retry_policy: [] + timeouts: null topic: my-topic module.pubsub.google_pubsub_topic.default: + kms_key_name: null + labels: null + message_retention_duration: null name: my-topic - project: my-project + project: project-id + timeouts: null counts: google_pubsub_subscription: 1 google_pubsub_topic: 1 + modules: 1 + resources: 2 diff --git a/tests/modules/pubsub/examples/schema.yaml b/tests/modules/pubsub/examples/schema.yaml index 7021d110..9e138529 100644 --- a/tests/modules/pubsub/examples/schema.yaml +++ b/tests/modules/pubsub/examples/schema.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,14 +16,23 @@ values: module.topic_with_schema.google_pubsub_schema.default[0]: definition: '{"fields":[{"name":"StringField","type":"string"},{"name":"FloatField","type":"float"},{"name":"BooleanField","type":"boolean"}],"name":"Avro","type":"record"}' name: my-topic-schema - project: my-project + project: project-id + timeouts: null type: AVRO module.topic_with_schema.google_pubsub_topic.default: + kms_key_name: null + labels: null + message_retention_duration: null name: my-topic - project: my-project + project: project-id schema_settings: - encoding: JSON + timeouts: null counts: google_pubsub_schema: 1 google_pubsub_topic: 1 + modules: 1 + resources: 2 + +outputs: {} diff --git a/tests/modules/pubsub/examples/simple.yaml b/tests/modules/pubsub/examples/simple.yaml index 6fe54ec6..88d9ac5e 100644 --- a/tests/modules/pubsub/examples/simple.yaml +++ b/tests/modules/pubsub/examples/simple.yaml @@ -14,23 +14,29 @@ values: module.pubsub.google_pubsub_topic.default: + kms_key_name: null + labels: null + message_retention_duration: null name: my-topic - project: my-project + project: project-id + timeouts: null module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.subscriber"]: condition: [] members: - - user:user1@example.com - project: my-project + - serviceAccount:service_account_email + project: project-id role: roles/pubsub.subscriber topic: my-topic module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.viewer"]: condition: [] members: - - group:foo@example.com - project: my-project + - group:organization-admins@example.org + project: project-id role: roles/pubsub.viewer topic: my-topic counts: google_pubsub_topic: 1 google_pubsub_topic_iam_binding: 2 + modules: 1 + resources: 3 \ No newline at end of file diff --git a/tests/modules/pubsub/examples/subscription-iam.yaml b/tests/modules/pubsub/examples/subscription-iam.yaml index 42ed2565..04b71ecb 100644 --- a/tests/modules/pubsub/examples/subscription-iam.yaml +++ b/tests/modules/pubsub/examples/subscription-iam.yaml @@ -12,16 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. + values: + module.pubsub.google_pubsub_subscription.default["test-1"]: + bigquery_config: [] + cloud_storage_config: [] + dead_letter_policy: [] + enable_exactly_once_delivery: false + enable_message_ordering: false + filter: null + labels: null + message_retention_duration: 604800s + name: test-1 + project: project-id + push_config: [] + retain_acked_messages: false + retry_policy: [] + timeouts: null + topic: my-topic module.pubsub.google_pubsub_subscription_iam_binding.authoritative["test-1.roles/pubsub.subscriber"]: condition: [] members: - - user:user1@example.com - project: my-project + - serviceAccount:service_account_email + project: project-id role: roles/pubsub.subscriber subscription: test-1 + module.pubsub.google_pubsub_topic.default: + kms_key_name: null + labels: null + message_retention_duration: null + name: my-topic + project: project-id + timeouts: null counts: google_pubsub_subscription: 1 google_pubsub_subscription_iam_binding: 1 google_pubsub_topic: 1 + modules: 1 + resources: 3 diff --git a/tests/modules/pubsub/examples/subscriptions.yaml b/tests/modules/pubsub/examples/subscriptions.yaml index b1a94212..89acf08b 100644 --- a/tests/modules/pubsub/examples/subscriptions.yaml +++ b/tests/modules/pubsub/examples/subscriptions.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,37 +15,51 @@ values: module.pubsub.google_pubsub_subscription.default["test-pull"]: bigquery_config: [] + cloud_storage_config: [] dead_letter_policy: [] - enable_exactly_once_delivery: False - enable_message_ordering: False + enable_exactly_once_delivery: false + enable_message_ordering: false filter: null labels: null message_retention_duration: 604800s name: test-pull - project: my-project + project: project-id push_config: [] - retain_acked_messages: False + retain_acked_messages: false retry_policy: [] + timeouts: null topic: my-topic module.pubsub.google_pubsub_subscription.default["test-pull-override"]: bigquery_config: [] + cloud_storage_config: [] dead_letter_policy: [] - enable_exactly_once_delivery: False - enable_message_ordering: False + effective_labels: + test: override + enable_exactly_once_delivery: false + enable_message_ordering: false filter: null labels: test: override message_retention_duration: 604800s name: test-pull-override - project: my-project + project: project-id push_config: [] - retain_acked_messages: True + retain_acked_messages: true retry_policy: [] + terraform_labels: + test: override + timeouts: null topic: my-topic module.pubsub.google_pubsub_topic.default: + kms_key_name: null + labels: null + message_retention_duration: null name: my-topic - project: my-project + project: project-id + timeouts: null counts: google_pubsub_subscription: 2 google_pubsub_topic: 1 + modules: 1 + resources: 3