From ad68fc4dfa576624a7e2caa1b96499161d8b0937 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 3 Mar 2021 14:19:08 +0100 Subject: [PATCH] Support for cloud logging buckets --- modules/folder/README.md | 24 ++- modules/folder/main.tf | 12 +- modules/folder/variables.tf | 2 + modules/logging-bucket/README.md | 61 ++++++ modules/logging-bucket/main.tf | 51 +++++ modules/logging-bucket/outputs.tf | 24 +++ modules/logging-bucket/variables.tf | 48 +++++ modules/organization/README.md | 24 ++- modules/organization/main.tf | 12 +- modules/organization/variables.tf | 2 + modules/project/README.md | 51 +++-- modules/project/main.tf | 19 +- modules/project/variables.tf | 11 +- tests/modules/folder/fixture/variables.tf | 1 + tests/modules/folder/test_plan_logging.py | 183 ++++++++++------ tests/modules/logging_bucket/__init__.py | 13 ++ tests/modules/logging_bucket/fixture/main.tf | 24 +++ .../logging_bucket/fixture/variables.tf | 42 ++++ tests/modules/logging_bucket/test_plan.py | 86 ++++++++ .../modules/organization/fixture/variables.tf | 1 + .../modules/organization/test_plan_logging.py | 181 ++++++++++------ tests/modules/project/fixture/variables.tf | 10 +- tests/modules/project/test_plan_logging.py | 199 ++++++++++++------ 23 files changed, 847 insertions(+), 234 deletions(-) create mode 100644 modules/logging-bucket/README.md create mode 100644 modules/logging-bucket/main.tf create mode 100644 modules/logging-bucket/outputs.tf create mode 100644 modules/logging-bucket/variables.tf create mode 100644 tests/modules/logging_bucket/__init__.py create mode 100644 tests/modules/logging_bucket/fixture/main.tf create mode 100644 tests/modules/logging_bucket/fixture/variables.tf create mode 100644 tests/modules/logging_bucket/test_plan.py diff --git a/modules/folder/README.md b/modules/folder/README.md index 98bbde94..586ad1e9 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -63,6 +63,13 @@ module "pubsub" { name = "pubsub_sink" } +module "bucket" { + source = "./modules/logging-bucket" + parent_type = "project" + parent = "my-project" + id = "bucket" +} + module "folder-sink" { source = "./modules/folder" parent = "folders/657104291943" @@ -74,6 +81,7 @@ module "folder-sink" { filter = "severity=WARNING" iam = false include_children = true + exclusions = {} } info = { type = "bigquery" @@ -81,6 +89,7 @@ module "folder-sink" { filter = "severity=INFO" iam = false include_children = true + exclusions = {} } notice = { type = "pubsub" @@ -88,13 +97,24 @@ module "folder-sink" { filter = "severity=NOTICE" iam = true include_children = true + exclusions = {} + } + debug = { + type = "logging" + destination = module.bucket.id + filter = "severity=DEBUG" + iam = true + include_children = true + exclusions = { + no-compute = "logName:compute" + } } } logging_exclusions = { no-gce-instances = "resource.type=gce_instance" } } -# tftest:modules=4:resources=9 +# tftest:modules=5:resources=11 ``` ### Hierarchical firewall policies @@ -151,7 +171,7 @@ module "folder2" { | *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | map(set(string)) | | {} | | *id* | Folder ID in case you use folder_create=false | string | | null | | *logging_exclusions* | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | -| *logging_sinks* | Logging sinks to create for this folder. | map(object({...})) | | {} | +| *logging_sinks* | Logging sinks to create for this folder. | map(object({...})) | | {} | | *name* | Folder name. | string | | null | | *parent* | Parent in folders/folder_id or organizations/org_id format. | string | | ... | | *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | diff --git a/modules/folder/main.tf b/modules/folder/main.tf index 154c8c3b..6d1c632c 100644 --- a/modules/folder/main.tf +++ b/modules/folder/main.tf @@ -30,8 +30,7 @@ locals { gcs = "storage.googleapis.com" bigquery = "bigquery.googleapis.com" pubsub = "pubsub.googleapis.com" - # TODO: add logging buckets support - # logging = "logging.googleapis.com" + logging = "logging.googleapis.com" } sink_bindings = { for type in ["gcs", "bigquery", "pubsub", "logging"] : @@ -192,6 +191,15 @@ resource "google_logging_folder_sink" "sink" { destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" filter = each.value.filter include_children = each.value.include_children + + dynamic "exclusions" { + for_each = each.value.exclusions + iterator = exclusion + content { + name = exclusion.key + filter = exclusion.value + } + } } resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" { diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf index b9d9b253..7726029b 100644 --- a/modules/folder/variables.tf +++ b/modules/folder/variables.tf @@ -83,6 +83,8 @@ variable "logging_sinks" { filter = string iam = bool include_children = bool + # TODO exclusions also support description and disabled + exclusions = map(string) })) default = {} } diff --git a/modules/logging-bucket/README.md b/modules/logging-bucket/README.md new file mode 100644 index 00000000..bd6d085b --- /dev/null +++ b/modules/logging-bucket/README.md @@ -0,0 +1,61 @@ +# Google Cloud Logging Buckets Module + +This module manages [logging buckets](https://cloud.google.com/logging/docs/storage#logs-buckets) for a project, folder, organization or billing account. + +Note that some logging buckets are automatically created for a given folder, project, organization, and billing account cannot be deleted. Creating a resource of this type will acquire and update the resource that already exists at the desired location. These buckets cannot be removed so deleting this resource will remove the bucket config from your terraform state but will leave the logging bucket unchanged. The buckets that are currently automatically created are "_Default" and "_Required". + +See also the `logging_sinks` argument within the [project](../project/), [folder](../folder/) and [organization](../organization) modules. + +## Examples + +### Create custom logging bucket in a project + +```hcl +module "bucket" { + source = "./modules/logging-bucket" + parent_type = "project" + parent = var.project_id + id = "mybucket" +} +# tftest:modules=1:resources=1 +``` + + +### Change retention period of a folder's _Default bucket + +```hcl +module "folder" { + source = "./modules/folder" + parent = "folders/657104291943" + name = "my folder" +} + +module "bucket-default" { + source = "./modules/logging-bucket" + parent_type = "folder" + parent = module.folder.id + id = "_Default" + retention = 10 +} +# tftest:modules=2:resources=2 +``` + + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| id | Name of the logging bucket. | string | ✓ | | +| parent | ID of the parentresource containing the bucket in the format 'project_id' 'folders/folder_id', 'organizations/organization_id' or 'billing_account_id'. | string | ✓ | | +| parent_type | Parent object type for the bucket (project, folder, organization, billing_account). | string | ✓ | | +| *description* | Human-readable description for the logging bucket. | string | | null | +| *location* | Location of the bucket. | string | | global | +| *retention* | Retention time in days for the logging bucket. | number | | 30 | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | None | | + diff --git a/modules/logging-bucket/main.tf b/modules/logging-bucket/main.tf new file mode 100644 index 00000000..ee405507 --- /dev/null +++ b/modules/logging-bucket/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_logging_project_bucket_config" "bucket" { + count = var.parent_type == "project" ? 1 : 0 + project = var.parent + location = var.location + retention_days = var.retention + bucket_id = var.id + description = var.description +} + +resource "google_logging_folder_bucket_config" "bucket" { + count = var.parent_type == "folder" ? 1 : 0 + folder = var.parent + location = var.location + retention_days = var.retention + bucket_id = var.id + description = var.description +} + +resource "google_logging_organization_bucket_config" "bucket" { + count = var.parent_type == "organization" ? 1 : 0 + organization = var.parent + location = var.location + retention_days = var.retention + bucket_id = var.id + description = var.description +} + +resource "google_logging_billing_account_bucket_config" "bucket" { + count = var.parent_type == "billing_account" ? 1 : 0 + billing_account = var.parent + location = var.location + retention_days = var.retention + bucket_id = var.id + description = var.description +} diff --git a/modules/logging-bucket/outputs.tf b/modules/logging-bucket/outputs.tf new file mode 100644 index 00000000..d0f9e3bf --- /dev/null +++ b/modules/logging-bucket/outputs.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + value = try( + google_logging_project_bucket_config.bucket.0.id, + google_logging_folder_bucket_config.bucket.0.id, + google_logging_organization_bucket_config.bucket.0.id, + google_logging_billing_account_bucket_config.bucket.0.id, + ) +} diff --git a/modules/logging-bucket/variables.tf b/modules/logging-bucket/variables.tf new file mode 100644 index 00000000..2a40c7c6 --- /dev/null +++ b/modules/logging-bucket/variables.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "parent_type" { + description = "Parent object type for the bucket (project, folder, organization, billing_account)." + type = string +} + +variable "parent" { + description = "ID of the parentresource containing the bucket in the format 'project_id' 'folders/folder_id', 'organizations/organization_id' or 'billing_account_id'." + type = string +} + +variable "location" { + description = "Location of the bucket." + type = string + default = "global" +} + +variable "id" { + description = "Name of the logging bucket." + type = string +} + +variable "description" { + description = "Human-readable description for the logging bucket." + type = string + default = null +} + +variable "retention" { + description = "Retention time in days for the logging bucket." + type = number + default = 30 +} diff --git a/modules/organization/README.md b/modules/organization/README.md index e16ab79a..f7b57f20 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -80,6 +80,13 @@ module "pubsub" { name = "pubsub_sink" } +module "bucket" { + source = "./modules/logging-bucket" + parent_type = "project" + parent = "my-project" + id = "bucket" +} + module "org" { source = "./modules/organization" organization_id = var.organization_id @@ -91,6 +98,7 @@ module "org" { filter = "severity=WARNING" iam = false include_children = true + exclusions = {} } info = { type = "bigquery" @@ -98,6 +106,7 @@ module "org" { filter = "severity=INFO" iam = false include_children = true + exclusions = {} } notice = { type = "pubsub" @@ -105,13 +114,24 @@ module "org" { filter = "severity=NOTICE" iam = true include_children = true + exclusions = {} + } + debug = { + type = "logging" + destination = module.bucket.id + filter = "severity=DEBUG" + iam = true + include_children = false + exclusions = { + no-compute = "logName:compute" + } } } logging_exclusions = { no-gce-instances = "resource.type=gce_instance" } } -# tftest:modules=4:resources=8 +# tftest:modules=5:resources=10 ``` @@ -132,7 +152,7 @@ module "org" { | *iam_audit_config_authoritative* | IAM Authoritative service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. Audit config should also be authoritative when using authoritative bindings. Use with caution. | map(map(list(string))) | | null | | *iam_bindings_authoritative* | IAM authoritative bindings, in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared. Bindings should also be authoritative when using authoritative audit config. Use with caution. | map(list(string)) | | null | | *logging_exclusions* | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | -| *logging_sinks* | Logging sinks to create for this organization. | map(object({...})) | | {} | +| *logging_sinks* | Logging sinks to create for this organization. | map(object({...})) | | {} | | *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | | *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({...})) | | {} | diff --git a/modules/organization/main.tf b/modules/organization/main.tf index 10788dba..e622e1ec 100644 --- a/modules/organization/main.tf +++ b/modules/organization/main.tf @@ -45,8 +45,7 @@ locals { gcs = "storage.googleapis.com" bigquery = "bigquery.googleapis.com" pubsub = "pubsub.googleapis.com" - # TODO: add logging buckets support - # logging = "logging.googleapis.com" + logging = "logging.googleapis.com" } sink_bindings = { for type in ["gcs", "bigquery", "pubsub", "logging"] : @@ -256,6 +255,15 @@ resource "google_logging_organization_sink" "sink" { destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" filter = each.value.filter include_children = each.value.include_children + + dynamic "exclusions" { + for_each = each.value.exclusions + iterator = exclusion + content { + name = exclusion.key + filter = exclusion.value + } + } } resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" { diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf index 67e1143b..1db13579 100644 --- a/modules/organization/variables.tf +++ b/modules/organization/variables.tf @@ -124,6 +124,8 @@ variable "logging_sinks" { filter = string iam = bool include_children = bool + # TODO exclusions also support description and disabled + exclusions = map(string) })) default = {} } diff --git a/modules/project/README.md b/modules/project/README.md index 050c3ffa..67832d76 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -94,6 +94,13 @@ module "pubsub" { name = "pubsub_sink" } +module "bucket" { + source = "./modules/logging-bucket" + parent_type = "project" + parent = "my-project" + id = "bucket" +} + module "project-host" { source = "./modules/project" name = "my-project" @@ -101,29 +108,45 @@ module "project-host" { parent = "folders/1234567890" logging_sinks = { warnings = { - type = "gcs" - destination = module.gcs.name - filter = "severity=WARNING" - iam = false + type = "gcs" + destination = module.gcs.name + filter = "severity=WARNING" + iam = false + unique_writer = false + exclusions = {} } info = { - type = "bigquery" - destination = module.dataset.id - filter = "severity=INFO" - iam = false + type = "bigquery" + destination = module.dataset.id + filter = "severity=INFO" + iam = false + unique_writer = false + exclusions = {} } notice = { - type = "pubsub" - destination = module.pubsub.id - filter = "severity=NOTICE" - iam = true + type = "pubsub" + destination = module.pubsub.id + filter = "severity=NOTICE" + iam = true + unique_writer = false + exclusions = {} + } + debug = { + type = "logging" + destination = module.bucket.id + filter = "severity=DEBUG" + iam = true + unique_writer = false + exclusions = { + no-compute = "logName:compute" + } } } logging_exclusions = { no-gce-instances = "resource.type=gce_instance" } } -# tftest:modules=4:resources=9 +# tftest:modules=5:resources=11 ``` @@ -143,7 +166,7 @@ module "project-host" { | *labels* | Resource labels. | map(string) | | {} | | *lien_reason* | If non-empty, creates a project lien with this description. | string | | | | *logging_exclusions* | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | -| *logging_sinks* | Logging sinks to create for this project. | map(object({...})) | | {} | +| *logging_sinks* | Logging sinks to create for this project. | map(object({...})) | | {} | | *oslogin* | Enable OS Login. | bool | | false | | *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | | *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | diff --git a/modules/project/main.tf b/modules/project/main.tf index 2a82f564..f0c7c446 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -42,8 +42,7 @@ locals { gcs = "storage.googleapis.com" bigquery = "bigquery.googleapis.com" pubsub = "pubsub.googleapis.com" - # TODO: add logging buckets support - # logging = "logging.googleapis.com" + logging = "logging.googleapis.com" } sink_bindings = { for type in ["gcs", "bigquery", "pubsub", "logging"] : @@ -263,9 +262,19 @@ resource "google_logging_project_sink" "sink" { for_each = local.logging_sinks name = each.key #description = "${each.key} (Terraform-managed)" - project = local.project.project_id - destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" - filter = each.value.filter + project = local.project.project_id + destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" + filter = each.value.filter + unique_writer_identity = each.value.unique_writer + + dynamic "exclusions" { + for_each = each.value.exclusions + iterator = exclusion + content { + name = exclusion.key + filter = exclusion.value + } + } } resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" { diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 64c9a08f..646fa4a2 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -169,10 +169,13 @@ variable "shared_vpc_service_config" { variable "logging_sinks" { description = "Logging sinks to create for this project." type = map(object({ - destination = string - type = string - filter = string - iam = bool + destination = string + type = string + filter = string + iam = bool + unique_writer = bool + # TODO exclusions also support description and disabled + exclusions = map(string) })) default = {} } diff --git a/tests/modules/folder/fixture/variables.tf b/tests/modules/folder/fixture/variables.tf index 85068de0..9d994bc0 100644 --- a/tests/modules/folder/fixture/variables.tf +++ b/tests/modules/folder/fixture/variables.tf @@ -61,6 +61,7 @@ variable "logging_sinks" { filter = string iam = bool include_children = bool + exclusions = map(string) })) default = {} } diff --git a/tests/modules/folder/test_plan_logging.py b/tests/modules/folder/test_plan_logging.py index 9f0169a1..d2e9edc4 100644 --- a/tests/modules/folder/test_plan_logging.py +++ b/tests/modules/folder/test_plan_logging.py @@ -18,18 +18,19 @@ import pytest from collections import Counter -FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture") def test_sinks(plan_runner): - "Test folder-level sinks." - logging_sinks = """ { + "Test folder-level sinks." + logging_sinks = """ { warning = { type = "gcs" destination = "mybucket" filter = "severity=WARNING" iam = true include_children = true + exclusions = {} } info = { type = "bigquery" @@ -37,6 +38,7 @@ def test_sinks(plan_runner): filter = "severity=INFO" iam = true include_children = true + exclusions = {} } notice = { type = "pubsub" @@ -44,72 +46,123 @@ def test_sinks(plan_runner): filter = "severity=NOTICE" iam = true include_children = false + exclusions = {} + } + debug = { + type = "logging" + destination = "projects/myproject/locations/global/buckets/mybucket" + filter = "severity=DEBUG" + iam = true + include_children = false + exclusions = { + no-compute = "logName:compute" + no-container = "logName:container" + } } } """ - _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) - assert len(resources) == 7 + _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) + assert len(resources) == 8 - resource_types = Counter([r['type'] for r in resources]) - assert resource_types == { - 'google_bigquery_dataset_iam_binding': 1, - 'google_folder': 1, - 'google_logging_folder_sink': 3, - 'google_pubsub_topic_iam_binding': 1, - 'google_storage_bucket_iam_binding': 1 - } - - sinks = [r for r in resources - if r['type'] == 'google_logging_folder_sink'] - assert sorted([r['index'] for r in sinks]) == [ - 'info', - 'notice', - 'warning', - ] - values = [(r['index'], r['values']['filter'], r['values']['destination'], - r['values']['include_children']) - for r in sinks] - assert sorted(values) == [ - ('info', - 'severity=INFO', - 'bigquery.googleapis.com/projects/myproject/datasets/mydataset', - True), - ('notice', - 'severity=NOTICE', - 'pubsub.googleapis.com/projects/myproject/topics/mytopic', - False), - ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket', True)] + resource_types = Counter([r["type"] for r in resources]) + assert resource_types == { + "google_bigquery_dataset_iam_binding": 1, + "google_folder": 1, + "google_logging_folder_sink": 4, + "google_pubsub_topic_iam_binding": 1, + "google_storage_bucket_iam_binding": 1, + } + + sinks = [r for r in resources if r["type"] == "google_logging_folder_sink"] + assert sorted([r["index"] for r in sinks]) == [ + "debug", + "info", + "notice", + "warning", + ] + values = [ + ( + r["index"], + r["values"]["filter"], + r["values"]["destination"], + r["values"]["include_children"], + ) + for r in sinks + ] + assert sorted(values) == [ + ( + "debug", + "severity=DEBUG", + "logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket", + False, + ), + ( + "info", + "severity=INFO", + "bigquery.googleapis.com/projects/myproject/datasets/mydataset", + True, + ), + ( + "notice", + "severity=NOTICE", + "pubsub.googleapis.com/projects/myproject/topics/mytopic", + False, + ), + ("warning", "severity=WARNING", "storage.googleapis.com/mybucket", True), + ] + + bindings = [r for r in resources if "binding" in r["type"]] + values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings] + assert sorted(values) == [ + ("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"), + ("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"), + ("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"), + ] + + exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks] + assert sorted(exclusions) == [ + ( + "debug", + [ + { + "description": None, + "disabled": False, + "filter": "logName:compute", + "name": "no-compute", + }, + { + "description": None, + "disabled": False, + "filter": "logName:container", + "name": "no-container", + }, + ], + ), + ("info", []), + ("notice", []), + ("warning", []), + ] - bindings = [r for r in resources - if 'binding' in r['type']] - values = [(r['index'], r['type'], r['values']['role']) - for r in bindings] - assert sorted(values) == [ - ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'), - ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'), - ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator') - ] - def test_exclusions(plan_runner): - "Test folder-level logging exclusions." - logging_exclusions = ( - '{' - 'exclusion1 = "resource.type=gce_instance", ' - 'exclusion2 = "severity=NOTICE", ' - '}' - ) - _, resources = plan_runner(FIXTURES_DIR, - logging_exclusions=logging_exclusions) - assert len(resources) == 3 - exclusions = [r for r in resources - if r['type'] == 'google_logging_folder_exclusion'] - assert sorted([r['index'] for r in exclusions]) == [ - 'exclusion1', - 'exclusion2', - ] - values = [(r['index'], r['values']['filter']) for r in exclusions] - assert sorted(values) == [ - ('exclusion1', 'resource.type=gce_instance'), - ('exclusion2', 'severity=NOTICE') - ] + "Test folder-level logging exclusions." + logging_exclusions = ( + "{" + 'exclusion1 = "resource.type=gce_instance", ' + 'exclusion2 = "severity=NOTICE", ' + "}" + ) + _, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions) + assert len(resources) == 3 + exclusions = [ + r for r in resources if r["type"] == "google_logging_folder_exclusion" + ] + assert sorted([r["index"] for r in exclusions]) == [ + "exclusion1", + "exclusion2", + ] + values = [(r["index"], r["values"]["filter"]) for r in exclusions] + assert sorted(values) == [ + ("exclusion1", "resource.type=gce_instance"), + ("exclusion2", "severity=NOTICE"), + ] diff --git a/tests/modules/logging_bucket/__init__.py b/tests/modules/logging_bucket/__init__.py new file mode 100644 index 00000000..d46dbae5 --- /dev/null +++ b/tests/modules/logging_bucket/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/modules/logging_bucket/fixture/main.tf b/tests/modules/logging_bucket/fixture/main.tf new file mode 100644 index 00000000..335bd782 --- /dev/null +++ b/tests/modules/logging_bucket/fixture/main.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test" { + source = "../../../../modules/logging-bucket" + parent_type = var.parent_type + parent = var.parent + id = var.id + retention = var.retention + location = var.location +} diff --git a/tests/modules/logging_bucket/fixture/variables.tf b/tests/modules/logging_bucket/fixture/variables.tf new file mode 100644 index 00000000..ef53e30d --- /dev/null +++ b/tests/modules/logging_bucket/fixture/variables.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "parent" { + type = string +} + +variable "parent_type" { + type = string + validation { + condition = contains(["project", "folder", "organization", "billing_account"], var.parent_type) + error_message = "Parent type must be project, folder, organization or billing_account." + } +} + +variable "location" { + type = string + default = "global" +} + +variable "id" { + type = string + default = "mybucket" +} + +variable "retention" { + type = number + default = 30 +} diff --git a/tests/modules/logging_bucket/test_plan.py b/tests/modules/logging_bucket/test_plan.py new file mode 100644 index 00000000..45793b8a --- /dev/null +++ b/tests/modules/logging_bucket/test_plan.py @@ -0,0 +1,86 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture") + + +def test_project_logging_bucket(plan_runner): + "Test project logging bucket." + _, resources = plan_runner(FIXTURES_DIR, parent_type="project", parent="myproject") + assert len(resources) == 1 + + resource = resources[0] + assert resource["type"] == "google_logging_project_bucket_config" + assert resource["values"] == { + "bucket_id": "mybucket", + "project": "myproject", + "location": "global", + "retention_days": 30, + } + + +def test_folder_logging_bucket(plan_runner): + "Test project logging bucket." + _, resources = plan_runner( + FIXTURES_DIR, parent_type="folder", parent="folders/0123456789" + ) + assert len(resources) == 1 + + resource = resources[0] + assert resource["type"] == "google_logging_folder_bucket_config" + assert resource["values"] == { + "bucket_id": "mybucket", + "folder": "folders/0123456789", + "location": "global", + "retention_days": 30, + } + + +def test_organization_logging_bucket(plan_runner): + "Test project logging bucket." + _, resources = plan_runner( + FIXTURES_DIR, parent_type="organization", parent="organizations/0123456789" + ) + assert len(resources) == 1 + + resource = resources[0] + assert resource["type"] == "google_logging_organization_bucket_config" + assert resource["values"] == { + "bucket_id": "mybucket", + "organization": "organizations/0123456789", + "location": "global", + "retention_days": 30, + } + + +def test_billing_account_logging_bucket(plan_runner): + "Test project logging bucket." + _, resources = plan_runner( + FIXTURES_DIR, parent_type="billing_account", parent="0123456789" + ) + assert len(resources) == 1 + + resource = resources[0] + assert resource["type"] == "google_logging_billing_account_bucket_config" + assert resource["values"] == { + "bucket_id": "mybucket", + "billing_account": "0123456789", + "location": "global", + "retention_days": 30, + } diff --git a/tests/modules/organization/fixture/variables.tf b/tests/modules/organization/fixture/variables.tf index 5eceff1c..43a86639 100644 --- a/tests/modules/organization/fixture/variables.tf +++ b/tests/modules/organization/fixture/variables.tf @@ -81,6 +81,7 @@ variable "logging_sinks" { filter = string iam = bool include_children = bool + exclusions = map(string) })) default = {} } diff --git a/tests/modules/organization/test_plan_logging.py b/tests/modules/organization/test_plan_logging.py index fd55a7f4..f6684d12 100644 --- a/tests/modules/organization/test_plan_logging.py +++ b/tests/modules/organization/test_plan_logging.py @@ -18,18 +18,19 @@ import pytest from collections import Counter -FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture") def test_sinks(plan_runner): - "Test folder-level sinks." - logging_sinks = """ { + "Test folder-level sinks." + logging_sinks = """ { warning = { type = "gcs" destination = "mybucket" filter = "severity=WARNING" iam = true include_children = true + exclusions = {} } info = { type = "bigquery" @@ -37,6 +38,7 @@ def test_sinks(plan_runner): filter = "severity=INFO" iam = true include_children = true + exclusions = {} } notice = { type = "pubsub" @@ -44,71 +46,122 @@ def test_sinks(plan_runner): filter = "severity=NOTICE" iam = true include_children = false + exclusions = {} + } + debug = { + type = "logging" + destination = "projects/myproject/locations/global/buckets/mybucket" + filter = "severity=DEBUG" + iam = true + include_children = false + exclusions = { + no-compute = "logName:compute" + no-container = "logName:container" + } } } """ - _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) - assert len(resources) == 6 + _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) + assert len(resources) == 7 - resource_types = Counter([r['type'] for r in resources]) - assert resource_types == { - 'google_bigquery_dataset_iam_binding': 1, - 'google_logging_organization_sink': 3, - 'google_pubsub_topic_iam_binding': 1, - 'google_storage_bucket_iam_binding': 1 - } - - sinks = [r for r in resources - if r['type'] == 'google_logging_organization_sink'] - assert sorted([r['index'] for r in sinks]) == [ - 'info', - 'notice', - 'warning', - ] - values = [(r['index'], r['values']['filter'], r['values']['destination'], - r['values']['include_children']) - for r in sinks] - assert sorted(values) == [ - ('info', - 'severity=INFO', - 'bigquery.googleapis.com/projects/myproject/datasets/mydataset', - True), - ('notice', - 'severity=NOTICE', - 'pubsub.googleapis.com/projects/myproject/topics/mytopic', - False), - ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket', True)] + resource_types = Counter([r["type"] for r in resources]) + assert resource_types == { + "google_bigquery_dataset_iam_binding": 1, + "google_logging_organization_sink": 4, + "google_pubsub_topic_iam_binding": 1, + "google_storage_bucket_iam_binding": 1, + } + + sinks = [r for r in resources if r["type"] == "google_logging_organization_sink"] + assert sorted([r["index"] for r in sinks]) == [ + "debug", + "info", + "notice", + "warning", + ] + values = [ + ( + r["index"], + r["values"]["filter"], + r["values"]["destination"], + r["values"]["include_children"], + ) + for r in sinks + ] + assert sorted(values) == [ + ( + "debug", + "severity=DEBUG", + "logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket", + False, + ), + ( + "info", + "severity=INFO", + "bigquery.googleapis.com/projects/myproject/datasets/mydataset", + True, + ), + ( + "notice", + "severity=NOTICE", + "pubsub.googleapis.com/projects/myproject/topics/mytopic", + False, + ), + ("warning", "severity=WARNING", "storage.googleapis.com/mybucket", True), + ] + + bindings = [r for r in resources if "binding" in r["type"]] + values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings] + assert sorted(values) == [ + ("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"), + ("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"), + ("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"), + ] + + exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks] + assert sorted(exclusions) == [ + ( + "debug", + [ + { + "description": None, + "disabled": False, + "filter": "logName:compute", + "name": "no-compute", + }, + { + "description": None, + "disabled": False, + "filter": "logName:container", + "name": "no-container", + }, + ], + ), + ("info", []), + ("notice", []), + ("warning", []), + ] - bindings = [r for r in resources - if 'binding' in r['type']] - values = [(r['index'], r['type'], r['values']['role']) - for r in bindings] - assert sorted(values) == [ - ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'), - ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'), - ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator') - ] - def test_exclusions(plan_runner): - "Test folder-level logging exclusions." - logging_exclusions = ( - '{' - 'exclusion1 = "resource.type=gce_instance", ' - 'exclusion2 = "severity=NOTICE", ' - '}' - ) - _, resources = plan_runner(FIXTURES_DIR, - logging_exclusions=logging_exclusions) - assert len(resources) == 2 - exclusions = [r for r in resources - if r['type'] == 'google_logging_organization_exclusion'] - assert sorted([r['index'] for r in exclusions]) == [ - 'exclusion1', - 'exclusion2', - ] - values = [(r['index'], r['values']['filter']) for r in exclusions] - assert sorted(values) == [ - ('exclusion1', 'resource.type=gce_instance'), - ('exclusion2', 'severity=NOTICE') - ] + "Test folder-level logging exclusions." + logging_exclusions = ( + "{" + 'exclusion1 = "resource.type=gce_instance", ' + 'exclusion2 = "severity=NOTICE", ' + "}" + ) + _, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions) + assert len(resources) == 2 + exclusions = [ + r for r in resources if r["type"] == "google_logging_organization_exclusion" + ] + assert sorted([r["index"] for r in exclusions]) == [ + "exclusion1", + "exclusion2", + ] + values = [(r["index"], r["values"]["filter"]) for r in exclusions] + assert sorted(values) == [ + ("exclusion1", "resource.type=gce_instance"), + ("exclusion2", "severity=NOTICE"), + ] diff --git a/tests/modules/project/fixture/variables.tf b/tests/modules/project/fixture/variables.tf index 8c4fe012..2417fe62 100644 --- a/tests/modules/project/fixture/variables.tf +++ b/tests/modules/project/fixture/variables.tf @@ -96,10 +96,12 @@ variable "services" { variable "logging_sinks" { type = map(object({ - destination = string - type = string - filter = string - iam = bool + destination = string + type = string + filter = string + iam = bool + exclusions = map(string) + unique_writer = bool })) default = {} } diff --git a/tests/modules/project/test_plan_logging.py b/tests/modules/project/test_plan_logging.py index 812277b5..6841d194 100644 --- a/tests/modules/project/test_plan_logging.py +++ b/tests/modules/project/test_plan_logging.py @@ -18,92 +18,151 @@ import pytest from collections import Counter -FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture") def test_sinks(plan_runner): - "Test folder-level sinks." - logging_sinks = """ { + "Test folder-level sinks." + logging_sinks = """ { warning = { - type = "gcs" - destination = "mybucket" - filter = "severity=WARNING" - iam = true + type = "gcs" + destination = "mybucket" + filter = "severity=WARNING" + iam = true + exclusions = {} + unique_writer = false } info = { type = "bigquery" destination = "projects/myproject/datasets/mydataset" filter = "severity=INFO" iam = true + exclusions = {} + unique_writer = false } notice = { - type = "pubsub" - destination = "projects/myproject/topics/mytopic" - filter = "severity=NOTICE" - iam = true + type = "pubsub" + destination = "projects/myproject/topics/mytopic" + filter = "severity=NOTICE" + iam = true + exclusions = {} + unique_writer = false + } + debug = { + type = "logging" + destination = "projects/myproject/locations/global/buckets/mybucket" + filter = "severity=DEBUG" + iam = true + exclusions = { + no-compute = "logName:compute" + no-container = "logName:container" + } + unique_writer = true } } """ - _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) - assert len(resources) == 7 + _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) + assert len(resources) == 8 - resource_types = Counter([r['type'] for r in resources]) - assert resource_types == { - 'google_bigquery_dataset_iam_binding': 1, - 'google_logging_project_sink': 3, - 'google_project': 1, - 'google_pubsub_topic_iam_binding': 1, - 'google_storage_bucket_iam_binding': 1 - } - - sinks = [r for r in resources - if r['type'] == 'google_logging_project_sink'] - assert sorted([r['index'] for r in sinks]) == [ - 'info', - 'notice', - 'warning', - ] - values = [(r['index'], r['values']['filter'], r['values']['destination']) - for r in sinks] - assert sorted(values) == [ - ('info', - 'severity=INFO', - 'bigquery.googleapis.com/projects/myproject/datasets/mydataset'), - ('notice', - 'severity=NOTICE', - 'pubsub.googleapis.com/projects/myproject/topics/mytopic'), - ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket')] + resource_types = Counter([r["type"] for r in resources]) + assert resource_types == { + "google_bigquery_dataset_iam_binding": 1, + "google_logging_project_sink": 4, + "google_project": 1, + "google_pubsub_topic_iam_binding": 1, + "google_storage_bucket_iam_binding": 1, + } + + sinks = [r for r in resources if r["type"] == "google_logging_project_sink"] + assert sorted([r["index"] for r in sinks]) == [ + "debug", + "info", + "notice", + "warning", + ] + values = [ + ( + r["index"], + r["values"]["filter"], + r["values"]["destination"], + r["values"]["unique_writer_identity"], + ) + for r in sinks + ] + assert sorted(values) == [ + ( + "debug", + "severity=DEBUG", + "logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket", + True, + ), + ( + "info", + "severity=INFO", + "bigquery.googleapis.com/projects/myproject/datasets/mydataset", + False, + ), + ( + "notice", + "severity=NOTICE", + "pubsub.googleapis.com/projects/myproject/topics/mytopic", + False, + ), + ("warning", "severity=WARNING", "storage.googleapis.com/mybucket", False), + ] + + bindings = [r for r in resources if "binding" in r["type"]] + values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings] + assert sorted(values) == [ + ("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"), + ("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"), + ("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"), + ] + + exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks] + assert sorted(exclusions) == [ + ( + "debug", + [ + { + "description": None, + "disabled": False, + "filter": "logName:compute", + "name": "no-compute", + }, + { + "description": None, + "disabled": False, + "filter": "logName:container", + "name": "no-container", + }, + ], + ), + ("info", []), + ("notice", []), + ("warning", []), + ] - bindings = [r for r in resources - if 'binding' in r['type']] - values = [(r['index'], r['type'], r['values']['role']) - for r in bindings] - assert sorted(values) == [ - ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'), - ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'), - ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator') - ] - def test_exclusions(plan_runner): - "Test folder-level logging exclusions." - logging_exclusions = ( - '{' - 'exclusion1 = "resource.type=gce_instance", ' - 'exclusion2 = "severity=NOTICE", ' - '}' - ) - _, resources = plan_runner(FIXTURES_DIR, - logging_exclusions=logging_exclusions) - assert len(resources) == 3 - exclusions = [r for r in resources - if r['type'] == 'google_logging_project_exclusion'] - assert sorted([r['index'] for r in exclusions]) == [ - 'exclusion1', - 'exclusion2', - ] - values = [(r['index'], r['values']['filter']) for r in exclusions] - assert sorted(values) == [ - ('exclusion1', 'resource.type=gce_instance'), - ('exclusion2', 'severity=NOTICE') - ] + "Test folder-level logging exclusions." + logging_exclusions = ( + "{" + 'exclusion1 = "resource.type=gce_instance", ' + 'exclusion2 = "severity=NOTICE", ' + "}" + ) + _, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions) + assert len(resources) == 3 + exclusions = [ + r for r in resources if r["type"] == "google_logging_project_exclusion" + ] + assert sorted([r["index"] for r in exclusions]) == [ + "exclusion1", + "exclusion2", + ] + values = [(r["index"], r["values"]["filter"]) for r in exclusions] + assert sorted(values) == [ + ("exclusion1", "resource.type=gce_instance"), + ("exclusion2", "severity=NOTICE"), + ]