feat: add e2e test for pubsub module (#2163)
* feat: add e2e test for pubsub module
This commit is contained in:
parent
9d474d1b6e
commit
688c6cfb79
|
@ -9,14 +9,14 @@ This module allows managing a single Pub/Sub topic, including multiple subscript
|
|||
```hcl
|
||||
module "pubsub" {
|
||||
source = "./fabric/modules/pubsub"
|
||||
project_id = "my-project"
|
||||
project_id = var.project_id
|
||||
name = "my-topic"
|
||||
iam = {
|
||||
"roles/pubsub.viewer" = ["group:foo@example.com"]
|
||||
"roles/pubsub.subscriber" = ["user:user1@example.com"]
|
||||
"roles/pubsub.viewer" = ["group:${var.group_email}"]
|
||||
"roles/pubsub.subscriber" = ["serviceAccount:${var.service_account.email}"]
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=3 inventory=simple.yaml
|
||||
# tftest modules=1 resources=3 inventory=simple.yaml e2e
|
||||
```
|
||||
|
||||
### Topic with schema
|
||||
|
@ -24,7 +24,7 @@ module "pubsub" {
|
|||
```hcl
|
||||
module "topic_with_schema" {
|
||||
source = "./fabric/modules/pubsub"
|
||||
project_id = "my-project"
|
||||
project_id = var.project_id
|
||||
name = "my-topic"
|
||||
schema = {
|
||||
msg_encoding = "JSON"
|
||||
|
@ -48,7 +48,7 @@ module "topic_with_schema" {
|
|||
})
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=2 inventory=schema.yaml
|
||||
# tftest modules=1 resources=2 inventory=schema.yaml e2e
|
||||
```
|
||||
|
||||
### Subscriptions
|
||||
|
@ -58,7 +58,7 @@ Subscriptions are defined with the `subscriptions` variable, allowing optional c
|
|||
```hcl
|
||||
module "pubsub" {
|
||||
source = "./fabric/modules/pubsub"
|
||||
project_id = "my-project"
|
||||
project_id = var.project_id
|
||||
name = "my-topic"
|
||||
subscriptions = {
|
||||
test-pull = {}
|
||||
|
@ -68,7 +68,7 @@ module "pubsub" {
|
|||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=3 inventory=subscriptions.yaml
|
||||
# tftest modules=1 resources=3 inventory=subscriptions.yaml e2e
|
||||
```
|
||||
|
||||
### Push subscriptions
|
||||
|
@ -78,7 +78,7 @@ Push subscriptions need extra configuration in the `push_configs` variable.
|
|||
```hcl
|
||||
module "pubsub" {
|
||||
source = "./fabric/modules/pubsub"
|
||||
project_id = "my-project"
|
||||
project_id = var.project_id
|
||||
name = "my-topic"
|
||||
subscriptions = {
|
||||
test-push = {
|
||||
|
@ -88,7 +88,7 @@ module "pubsub" {
|
|||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=2
|
||||
# tftest modules=1 resources=2 inventory=push-subscription.yaml e2e
|
||||
```
|
||||
|
||||
### BigQuery subscriptions
|
||||
|
@ -98,12 +98,12 @@ BigQuery subscriptions need extra configuration in the `bigquery_subscription_co
|
|||
```hcl
|
||||
module "pubsub" {
|
||||
source = "./fabric/modules/pubsub"
|
||||
project_id = "my-project"
|
||||
project_id = var.project_id
|
||||
name = "my-topic"
|
||||
subscriptions = {
|
||||
test-bigquery = {
|
||||
bigquery = {
|
||||
table = "my_project_id:my_dataset.my_table"
|
||||
table = "${module.bigquery-dataset.tables["my_table"].project}:${module.bigquery-dataset.tables["my_table"].dataset_id}.${module.bigquery-dataset.tables["my_table"].table_id}"
|
||||
use_topic_schema = true
|
||||
write_metadata = false
|
||||
drop_unknown_fields = true
|
||||
|
@ -111,7 +111,7 @@ module "pubsub" {
|
|||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=2
|
||||
# tftest modules=2 resources=5 fixtures=fixtures/bigquery-dataset.tf inventory=bigquery-subscription.yaml e2e
|
||||
```
|
||||
|
||||
### Cloud Storage subscriptions
|
||||
|
@ -121,13 +121,13 @@ Cloud Storage subscriptions need extra configuration in the `cloud_storage_subsc
|
|||
```hcl
|
||||
module "pubsub" {
|
||||
source = "./fabric/modules/pubsub"
|
||||
project_id = "my-project"
|
||||
project_id = var.project_id
|
||||
name = "my-topic"
|
||||
subscriptions = {
|
||||
test-cloudstorage = {
|
||||
cloud_storage = {
|
||||
bucket = "my-bucket"
|
||||
filename_prefix = "test_prefix"
|
||||
bucket = module.gcs.name
|
||||
filename_prefix = var.prefix
|
||||
filename_suffix = "test_suffix"
|
||||
max_duration = "100s"
|
||||
max_bytes = 1000
|
||||
|
@ -138,24 +138,24 @@ module "pubsub" {
|
|||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=2
|
||||
# tftest modules=2 resources=4 fixtures=fixtures/gcs.tf inventory=cloud-storage-subscription.yaml e2e
|
||||
```
|
||||
### Subscriptions with IAM
|
||||
|
||||
```hcl
|
||||
module "pubsub" {
|
||||
source = "./fabric/modules/pubsub"
|
||||
project_id = "my-project"
|
||||
project_id = var.project_id
|
||||
name = "my-topic"
|
||||
subscriptions = {
|
||||
test-1 = {
|
||||
iam = {
|
||||
"roles/pubsub.subscriber" = ["user:user1@example.com"]
|
||||
"roles/pubsub.subscriber" = ["serviceAccount:${var.service_account.email}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=3
|
||||
# tftest modules=1 resources=3 inventory=subscription-iam.yaml e2e
|
||||
```
|
||||
<!-- BEGIN TFDOC -->
|
||||
## Variables
|
||||
|
@ -184,4 +184,9 @@ module "pubsub" {
|
|||
| [subscription_id](outputs.tf#L37) | Subscription ids. | |
|
||||
| [subscriptions](outputs.tf#L48) | Subscription resources. | |
|
||||
| [topic](outputs.tf#L57) | Topic resource. | |
|
||||
|
||||
## Fixtures
|
||||
|
||||
- [bigquery-dataset.tf](../../tests/fixtures/bigquery-dataset.tf)
|
||||
- [gcs.tf](../../tests/fixtures/gcs.tf)
|
||||
<!-- END TFDOC -->
|
||||
|
|
|
@ -0,0 +1,38 @@
|
|||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
locals {
|
||||
schema = jsonencode([
|
||||
{ name = "data", type = "STRING" },
|
||||
{ name = "freq", type = "INT64" }
|
||||
])
|
||||
}
|
||||
|
||||
module "bigquery-dataset" {
|
||||
source = "./fabric/modules/bigquery-dataset"
|
||||
project_id = var.project_id
|
||||
id = "my_dataset"
|
||||
tables = {
|
||||
my_table = {
|
||||
deletion_protection = false
|
||||
schema = local.schema
|
||||
partitioning = {
|
||||
time = { type = "DAY", expiration_ms = null }
|
||||
}
|
||||
}
|
||||
}
|
||||
iam = {
|
||||
"roles/bigquery.dataEditor" = ["serviceAccount:service-${var.project_number}@gcp-sa-pubsub.iam.gserviceaccount.com"]
|
||||
}
|
||||
}
|
|
@ -0,0 +1,23 @@
|
|||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
module "gcs" {
|
||||
source = "./fabric/modules/gcs"
|
||||
project_id = var.project_id
|
||||
prefix = var.prefix
|
||||
name = "my-bucket"
|
||||
iam = {
|
||||
"roles/storage.admin" = ["serviceAccount:service-${var.project_number}@gcp-sa-pubsub.iam.gserviceaccount.com"]
|
||||
}
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright 2023 Google LLC
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -16,16 +16,21 @@ values:
|
|||
module.pubsub.google_pubsub_subscription.default["test-bigquery"]:
|
||||
bigquery_config:
|
||||
- drop_unknown_fields: true
|
||||
table: my_project_id:my_dataset.my_table
|
||||
table: project-id:my_dataset.my_table
|
||||
use_topic_schema: true
|
||||
write_metadata: false
|
||||
name: test-bigquery
|
||||
project: my-project
|
||||
project: project-id
|
||||
topic: my-topic
|
||||
module.pubsub.google_pubsub_topic.default:
|
||||
name: my-topic
|
||||
project: my-project
|
||||
project: project-id
|
||||
|
||||
counts:
|
||||
google_bigquery_dataset: 1
|
||||
google_bigquery_dataset_iam_binding: 1
|
||||
google_bigquery_table: 1
|
||||
google_pubsub_subscription: 1
|
||||
google_pubsub_topic: 1
|
||||
modules: 2
|
||||
resources: 5
|
||||
|
|
|
@ -0,0 +1,53 @@
|
|||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
values:
|
||||
module.pubsub.google_pubsub_subscription.default["test-cloudstorage"]:
|
||||
bigquery_config: []
|
||||
cloud_storage_config:
|
||||
- avro_config:
|
||||
- write_metadata: true
|
||||
bucket: test-my-bucket
|
||||
filename_prefix: test
|
||||
filename_suffix: test_suffix
|
||||
max_bytes: 1000
|
||||
max_duration: 100s
|
||||
dead_letter_policy: []
|
||||
enable_exactly_once_delivery: false
|
||||
enable_message_ordering: false
|
||||
filter: null
|
||||
labels: null
|
||||
message_retention_duration: 604800s
|
||||
name: test-cloudstorage
|
||||
project: project-id
|
||||
push_config: []
|
||||
retain_acked_messages: false
|
||||
retry_policy: []
|
||||
timeouts: null
|
||||
topic: my-topic
|
||||
module.pubsub.google_pubsub_topic.default:
|
||||
kms_key_name: null
|
||||
labels: null
|
||||
message_retention_duration: null
|
||||
name: my-topic
|
||||
project: project-id
|
||||
timeouts: null
|
||||
|
||||
counts:
|
||||
google_pubsub_subscription: 1
|
||||
google_pubsub_topic: 1
|
||||
google_storage_bucket: 1
|
||||
google_storage_bucket_iam_binding: 1
|
||||
modules: 2
|
||||
resources: 4
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright 2023 Google LLC
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -12,19 +12,38 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
values:
|
||||
module.pubsub.google_pubsub_subscription.default["test-push"]:
|
||||
bigquery_config: []
|
||||
cloud_storage_config: []
|
||||
dead_letter_policy: []
|
||||
enable_exactly_once_delivery: false
|
||||
enable_message_ordering: false
|
||||
filter: null
|
||||
labels: null
|
||||
message_retention_duration: 604800s
|
||||
name: test-push
|
||||
project: my-project
|
||||
project: project-id
|
||||
push_config:
|
||||
- attributes: null
|
||||
no_wrapper: []
|
||||
oidc_token: []
|
||||
push_endpoint: https://example.com/foo
|
||||
retain_acked_messages: false
|
||||
retry_policy: []
|
||||
timeouts: null
|
||||
topic: my-topic
|
||||
module.pubsub.google_pubsub_topic.default:
|
||||
kms_key_name: null
|
||||
labels: null
|
||||
message_retention_duration: null
|
||||
name: my-topic
|
||||
project: my-project
|
||||
project: project-id
|
||||
timeouts: null
|
||||
|
||||
counts:
|
||||
google_pubsub_subscription: 1
|
||||
google_pubsub_topic: 1
|
||||
modules: 1
|
||||
resources: 2
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright 2023 Google LLC
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -16,14 +16,23 @@ values:
|
|||
module.topic_with_schema.google_pubsub_schema.default[0]:
|
||||
definition: '{"fields":[{"name":"StringField","type":"string"},{"name":"FloatField","type":"float"},{"name":"BooleanField","type":"boolean"}],"name":"Avro","type":"record"}'
|
||||
name: my-topic-schema
|
||||
project: my-project
|
||||
project: project-id
|
||||
timeouts: null
|
||||
type: AVRO
|
||||
module.topic_with_schema.google_pubsub_topic.default:
|
||||
kms_key_name: null
|
||||
labels: null
|
||||
message_retention_duration: null
|
||||
name: my-topic
|
||||
project: my-project
|
||||
project: project-id
|
||||
schema_settings:
|
||||
- encoding: JSON
|
||||
timeouts: null
|
||||
|
||||
counts:
|
||||
google_pubsub_schema: 1
|
||||
google_pubsub_topic: 1
|
||||
modules: 1
|
||||
resources: 2
|
||||
|
||||
outputs: {}
|
||||
|
|
|
@ -14,23 +14,29 @@
|
|||
|
||||
values:
|
||||
module.pubsub.google_pubsub_topic.default:
|
||||
kms_key_name: null
|
||||
labels: null
|
||||
message_retention_duration: null
|
||||
name: my-topic
|
||||
project: my-project
|
||||
project: project-id
|
||||
timeouts: null
|
||||
module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.subscriber"]:
|
||||
condition: []
|
||||
members:
|
||||
- user:user1@example.com
|
||||
project: my-project
|
||||
- serviceAccount:service_account_email
|
||||
project: project-id
|
||||
role: roles/pubsub.subscriber
|
||||
topic: my-topic
|
||||
module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.viewer"]:
|
||||
condition: []
|
||||
members:
|
||||
- group:foo@example.com
|
||||
project: my-project
|
||||
- group:organization-admins@example.org
|
||||
project: project-id
|
||||
role: roles/pubsub.viewer
|
||||
topic: my-topic
|
||||
|
||||
counts:
|
||||
google_pubsub_topic: 1
|
||||
google_pubsub_topic_iam_binding: 2
|
||||
modules: 1
|
||||
resources: 3
|
|
@ -12,16 +12,42 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
values:
|
||||
module.pubsub.google_pubsub_subscription.default["test-1"]:
|
||||
bigquery_config: []
|
||||
cloud_storage_config: []
|
||||
dead_letter_policy: []
|
||||
enable_exactly_once_delivery: false
|
||||
enable_message_ordering: false
|
||||
filter: null
|
||||
labels: null
|
||||
message_retention_duration: 604800s
|
||||
name: test-1
|
||||
project: project-id
|
||||
push_config: []
|
||||
retain_acked_messages: false
|
||||
retry_policy: []
|
||||
timeouts: null
|
||||
topic: my-topic
|
||||
module.pubsub.google_pubsub_subscription_iam_binding.authoritative["test-1.roles/pubsub.subscriber"]:
|
||||
condition: []
|
||||
members:
|
||||
- user:user1@example.com
|
||||
project: my-project
|
||||
- serviceAccount:service_account_email
|
||||
project: project-id
|
||||
role: roles/pubsub.subscriber
|
||||
subscription: test-1
|
||||
module.pubsub.google_pubsub_topic.default:
|
||||
kms_key_name: null
|
||||
labels: null
|
||||
message_retention_duration: null
|
||||
name: my-topic
|
||||
project: project-id
|
||||
timeouts: null
|
||||
|
||||
counts:
|
||||
google_pubsub_subscription: 1
|
||||
google_pubsub_subscription_iam_binding: 1
|
||||
google_pubsub_topic: 1
|
||||
modules: 1
|
||||
resources: 3
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright 2023 Google LLC
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -15,37 +15,51 @@
|
|||
values:
|
||||
module.pubsub.google_pubsub_subscription.default["test-pull"]:
|
||||
bigquery_config: []
|
||||
cloud_storage_config: []
|
||||
dead_letter_policy: []
|
||||
enable_exactly_once_delivery: False
|
||||
enable_message_ordering: False
|
||||
enable_exactly_once_delivery: false
|
||||
enable_message_ordering: false
|
||||
filter: null
|
||||
labels: null
|
||||
message_retention_duration: 604800s
|
||||
name: test-pull
|
||||
project: my-project
|
||||
project: project-id
|
||||
push_config: []
|
||||
retain_acked_messages: False
|
||||
retain_acked_messages: false
|
||||
retry_policy: []
|
||||
timeouts: null
|
||||
topic: my-topic
|
||||
module.pubsub.google_pubsub_subscription.default["test-pull-override"]:
|
||||
bigquery_config: []
|
||||
cloud_storage_config: []
|
||||
dead_letter_policy: []
|
||||
enable_exactly_once_delivery: False
|
||||
enable_message_ordering: False
|
||||
effective_labels:
|
||||
test: override
|
||||
enable_exactly_once_delivery: false
|
||||
enable_message_ordering: false
|
||||
filter: null
|
||||
labels:
|
||||
test: override
|
||||
message_retention_duration: 604800s
|
||||
name: test-pull-override
|
||||
project: my-project
|
||||
project: project-id
|
||||
push_config: []
|
||||
retain_acked_messages: True
|
||||
retain_acked_messages: true
|
||||
retry_policy: []
|
||||
terraform_labels:
|
||||
test: override
|
||||
timeouts: null
|
||||
topic: my-topic
|
||||
module.pubsub.google_pubsub_topic.default:
|
||||
kms_key_name: null
|
||||
labels: null
|
||||
message_retention_duration: null
|
||||
name: my-topic
|
||||
project: my-project
|
||||
project: project-id
|
||||
timeouts: null
|
||||
|
||||
counts:
|
||||
google_pubsub_subscription: 2
|
||||
google_pubsub_topic: 1
|
||||
modules: 1
|
||||
resources: 3
|
||||
|
|
Loading…
Reference in New Issue