diff --git a/CHANGELOG.md b/CHANGELOG.md index 8441b132..fdd77ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] -- add support for creating logging sinks and logging exclusions in the `project`, `folder` and `organization` modules +- **incompatible change** removed the `logging-sinks` module. Logging sinks can now be created the `logging_sinks` variable in the in the `project`, `folder` and `organization` modules +- add support for creating logging exclusions in the `project`, `folder` and `organization` modules - add support for Confidential Compute to `compute-vm` module ## [4.2.0] - 2020-11-25 diff --git a/README.md b/README.md index fa2d2864..fd39b431 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The current list of modules supports most of the core foundational and networkin Currently available modules: -- **foundational** - [folder](./modules/folder), [log sinks](./modules/logging-sinks), [organization](./modules/organization), [project](./modules/project), [service accounts](./modules/iam-service-account) +- **foundational** - [folder](./modules/folder), [organization](./modules/organization), [project](./modules/project), [service accounts](./modules/iam-service-account) - **networking** - [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN static](./modules/net-vpn-static), [VPN dynamic](./modules/net-vpn-dynamic), [VPN HA](./modules/net-vpn-ha), [NAT](./modules/net-cloudnat), [address reservation](./modules/net-address), [DNS](./modules/dns), [L4 ILB](./modules/net-ilb), [Service Directory](./modules/service-directory), [Cloud Endpoints](./modules/cloudenpoints) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [GKE cluster](./modules/gke-cluster), [GKE nodepool](./modules/gke-nodepool), [COS container](./modules/cos-container) (coredns, mysql, onprem, squid) - **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion), [Bigtable instance](./modules/bigtable-instance) diff --git a/foundations/business-units/main.tf b/foundations/business-units/main.tf index 9f7c9b4d..d00acaf9 100644 --- a/foundations/business-units/main.tf +++ b/foundations/business-units/main.tf @@ -14,6 +14,19 @@ * limitations under the License. */ +locals { + logging_sinks = { + audit-logs = { + type = "bigquery" + destination = module.audit-dataset.id + filter = var.audit_filter + iam = true + include_children = true + } + } + root_node_type = split("/", var.root_node)[0] +} + ############################################################################### # Terraform top-level resources # ############################################################################### @@ -99,8 +112,7 @@ module "audit-project" { prefix = var.prefix billing_account = var.billing_account_id iam = { - "roles/bigquery.dataEditor" = [module.audit-log-sinks.writer_identities[0]] - "roles/viewer" = var.iam_audit_viewers + "roles/viewer" = var.iam_audit_viewers } services = concat(var.project_services, [ "bigquery.googleapis.com", @@ -122,16 +134,22 @@ module "audit-dataset" { } } -module "audit-log-sinks" { - source = "../../modules/logging-sinks" - parent = var.root_node - destinations = { - audit-logs = "bigquery.googleapis.com/${module.audit-dataset.id}" - } - sinks = { - audit-logs = var.audit_filter - } -} +# uncomment the next two modules to create the logging sinks + +# module "root_org" { +# count = local.root_node_type == "organizations" ? 1 : 0 +# source = "../../modules/organization" +# organization_id = var.root_node +# logging_sinks = local.logging_sinks +# } + +# module "root_folder" { +# count = local.root_node_type == "folders" ? 1 : 0 +# source = "../../modules/folder" +# id = var.root_node +# folder_create = false +# logging_sinks = local.logging_sinks +# } ############################################################################### # Shared resources (GCR, GCS, KMS, etc.) # diff --git a/foundations/environments/locals.tf b/foundations/environments/locals.tf index 1f0a4ae7..50becb64 100644 --- a/foundations/environments/locals.tf +++ b/foundations/environments/locals.tf @@ -36,4 +36,14 @@ locals { || substr(var.root_node, 0, 13) == "organizations" ) + logging_sinks = { + audit-logs = { + type = "bigquery" + destination = module.audit-dataset.id + filter = var.audit_filter + iam = true + include_children = true + } + } + root_node_type = split("/", var.root_node)[0] } diff --git a/foundations/environments/main.tf b/foundations/environments/main.tf index 48ac7568..8b92fcbe 100644 --- a/foundations/environments/main.tf +++ b/foundations/environments/main.tf @@ -105,8 +105,7 @@ module "audit-project" { prefix = var.prefix billing_account = var.billing_account_id iam = { - "roles/bigquery.dataEditor" = [module.audit-log-sinks.writer_identities[0]] - "roles/viewer" = var.iam_audit_viewers + "roles/viewer" = var.iam_audit_viewers } services = concat(var.project_services, [ "bigquery.googleapis.com", @@ -128,16 +127,23 @@ module "audit-dataset" { } } -module "audit-log-sinks" { - source = "../../modules/logging-sinks" - parent = var.root_node - destinations = { - audit-logs = "bigquery.googleapis.com/${module.audit-dataset.id}" - } - sinks = { - audit-logs = var.audit_filter - } -} +# uncomment the next two modules to create the logging sinks + +# module "root_org" { +# count = local.root_node_type == "organizations" ? 1 : 0 +# source = "../../modules/organization" +# organization_id = var.root_node +# logging_sinks = local.logging_sinks +# } + +# module "root_folder" { +# count = local.root_node_type == "folders" ? 1 : 0 +# source = "../../modules/folder" +# id = var.root_node +# folder_create = false +# logging_sinks = local.logging_sinks +# } + ############################################################################### # Shared resources (GCR, GCS, KMS, etc.) # diff --git a/modules/README.md b/modules/README.md index c9ab6f2d..a86196f5 100644 --- a/modules/README.md +++ b/modules/README.md @@ -11,7 +11,6 @@ Specific modules also offer support for non-authoritative bindings (e.g. `google ## Foundational modules - [folder](./folder) -- [log sinks](./logging-sinks) - [organization](./organization) - [project](./project) - [service account](./iam-service-account) diff --git a/modules/folder/README.md b/modules/folder/README.md index dede3fd2..3e7e6597 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -69,22 +69,25 @@ module "folder-sink" { name = "my-folder" logging_sinks = { warnings = { - type = "gcs" - destination = module.gcs.name - filter = "severity=WARNING" - grant = false + type = "gcs" + destination = module.gcs.name + filter = "severity=WARNING" + iam = false + include_children = true } info = { - type = "bigquery" - destination = module.dataset.id - filter = "severity=INFO" - grant = false + type = "bigquery" + destination = module.dataset.id + filter = "severity=INFO" + iam = false + include_children = true } notice = { - type = "pubsub" - destination = module.pubsub.id - filter = "severity=NOTICE" - grant = true + type = "pubsub" + destination = module.pubsub.id + filter = "severity=NOTICE" + iam = true + include_children = true } } logging_exclusions = { @@ -147,7 +150,7 @@ module "folder2" { | *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | map(set(string)) | | {} | | *id* | Folder ID in case you use folder_create=false | string | | null | | *logging_exclusions* | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | -| *logging_sinks* | Logging sinks to create for this folder. | map(object({...})) | | {} | +| *logging_sinks* | Logging sinks to create for this folder. | map(object({...})) | | {} | | *name* | Folder name. | string | | null | | *parent* | Parent in folders/folder_id or organizations/org_id format. | string | | ... | | *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | diff --git a/modules/folder/main.tf b/modules/folder/main.tf index b46eed4f..5b9ca936 100644 --- a/modules/folder/main.tf +++ b/modules/folder/main.tf @@ -38,7 +38,7 @@ locals { type => { for name, sink in local.logging_sinks : name => sink - if sink.grant && sink.type == type + if sink.iam && sink.type == type } } folder = ( @@ -188,9 +188,10 @@ resource "google_logging_folder_sink" "sink" { for_each = local.logging_sinks name = each.key #description = "${each.key} (Terraform-managed)" - folder = local.folder.name - destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" - filter = each.value.filter + folder = local.folder.name + destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" + filter = each.value.filter + include_children = each.value.include_children } resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" { @@ -216,13 +217,6 @@ resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" { members = [google_logging_folder_sink.sink[each.key].writer_identity] } -# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" { -# for_each = local.sink_grants["gcs"] -# bucket = each.value.destination -# role = "roles/storage.objectCreator" -# members = [google_logging_folder_sink.sink[each.key].writer_identity] -# } - resource "google_logging_folder_exclusion" "logging-exclusion" { for_each = coalesce(var.logging_exclusions, {}) name = each.key diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf index fc5ff7eb..eac4ebc0 100644 --- a/modules/folder/variables.tf +++ b/modules/folder/variables.tf @@ -78,10 +78,11 @@ variable "firewall_policy_attachments" { variable "logging_sinks" { description = "Logging sinks to create for this folder." type = map(object({ - destination = string - type = string - filter = string - grant = bool + destination = string + type = string + filter = string + iam = bool + include_children = bool })) default = {} } diff --git a/modules/logging-sinks/README.md b/modules/logging-sinks/README.md deleted file mode 100644 index 83f50165..00000000 --- a/modules/logging-sinks/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Terraform Logging Sinks Module - -This module allows easy creation of one or more logging sinks. - -## Example - - -## Variables - -| name | description | type | required | default | -|---|---|:---: |:---:|:---:| -| destinations | Map of destinations by sink name. | map(string) | ✓ | | -| parent | Resource where the sink will be created, eg 'organizations/nnnnnnnn'. | string | ✓ | | -| sinks | Map of sink name / sink filter. | map(string) | ✓ | | -| *default_options* | Default options used for sinks where no specific options are set. | object({...}) | | ... | -| *sink_options* | Optional map of sink name / sink options. If no options are specified for a sink defaults will be used. | map(object({...})) | | {} | - -## Outputs - -| name | description | sensitive | -|---|---|:---:| -| names | Log sink names. | | -| sinks | Log sink resources. | | -| writer_identities | Log sink writer identities. | | - - diff --git a/modules/logging-sinks/main.tf b/modules/logging-sinks/main.tf deleted file mode 100644 index 927bcae2..00000000 --- a/modules/logging-sinks/main.tf +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - bigquery_destinations = { - for name, destination in var.destinations : - name => substr(destination, 0, 8) == "bigquery" - } - resource_type = element(split("/", var.parent), 0) - resource_id = element(split("/", var.parent), 1) - sink_options = { - for name, _ in var.sinks : - name => lookup(var.sink_options, name, var.default_options) - } - sink_resources = concat( - [for _, sink in google_logging_organization_sink.sinks : sink], - [for _, sink in google_logging_billing_account_sink.sinks : sink], - [for _, sink in google_logging_folder_sink.sinks : sink], - [for _, sink in google_logging_project_sink.sinks : sink], - ) -} - -resource "google_logging_organization_sink" "sinks" { - for_each = local.resource_type == "organizations" ? var.sinks : {} - name = each.key - org_id = local.resource_id - filter = each.value - destination = var.destinations[each.key] - include_children = local.sink_options[each.key].include_children - dynamic bigquery_options { - for_each = local.bigquery_destinations[each.key] ? ["1"] : [] - iterator = config - content { - use_partitioned_tables = local.sink_options[each.key].bigquery_partitioned_tables - } - } -} - -resource "google_logging_billing_account_sink" "sinks" { - for_each = local.resource_type == "billing_accounts" ? var.sinks : {} - name = each.key - billing_account = local.resource_id - filter = each.value - destination = var.destinations[each.key] - dynamic bigquery_options { - for_each = local.bigquery_destinations[each.key] ? ["1"] : [] - iterator = config - content { - use_partitioned_tables = local.sink_options[each.key].bigquery_partitioned_tables - } - } -} - -resource "google_logging_folder_sink" "sinks" { - for_each = local.resource_type == "folders" ? var.sinks : {} - name = each.key - folder = var.parent - filter = each.value - destination = var.destinations[each.key] - include_children = local.sink_options[each.key].include_children - dynamic bigquery_options { - for_each = local.bigquery_destinations[each.key] ? ["1"] : [] - iterator = config - content { - use_partitioned_tables = local.sink_options[each.key].bigquery_partitioned_tables - } - } -} - -resource "google_logging_project_sink" "sinks" { - for_each = local.resource_type == "projects" ? var.sinks : {} - name = each.key - project = local.resource_id - filter = each.value - destination = var.destinations[each.key] - unique_writer_identity = local.sink_options[each.key].unique_writer_identity - dynamic bigquery_options { - for_each = local.bigquery_destinations[each.key] ? ["1"] : [] - iterator = config - content { - use_partitioned_tables = local.sink_options[each.key].bigquery_partitioned_tables - } - } -} diff --git a/modules/logging-sinks/outputs.tf b/modules/logging-sinks/outputs.tf deleted file mode 100644 index 900d02f0..00000000 --- a/modules/logging-sinks/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "sinks" { - description = "Log sink resources." - value = local.sink_resources -} - -output "names" { - description = "Log sink names." - value = [for sink in local.sink_resources : sink.name] -} - -output "writer_identities" { - description = "Log sink writer identities." - value = [for sink in local.sink_resources : sink.writer_identity] -} diff --git a/modules/logging-sinks/variables.tf b/modules/logging-sinks/variables.tf deleted file mode 100644 index 6ca20d4a..00000000 --- a/modules/logging-sinks/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "default_options" { - description = "Default options used for sinks where no specific options are set." - type = object({ - bigquery_partitioned_tables = bool - include_children = bool - unique_writer_identity = bool - }) - default = { - bigquery_partitioned_tables = true - include_children = true - unique_writer_identity = false - } -} - -variable "destinations" { - description = "Map of destinations by sink name." - type = map(string) -} - -variable "parent" { - description = "Resource where the sink will be created, eg 'organizations/nnnnnnnn'." - type = string -} - -variable "sink_options" { - description = "Optional map of sink name / sink options. If no options are specified for a sink defaults will be used." - type = map(object({ - bigquery_partitioned_tables = bool - include_children = bool - unique_writer_identity = bool - })) - default = {} -} - -variable "sinks" { - description = "Map of sink name / sink filter." - type = map(string) -} diff --git a/modules/logging-sinks/versions.tf b/modules/logging-sinks/versions.tf deleted file mode 100644 index bc4c2a9d..00000000 --- a/modules/logging-sinks/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.12.6" -} diff --git a/modules/organization/README.md b/modules/organization/README.md index f8b500fb..f51d4365 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -86,22 +86,25 @@ module "org" { logging_sinks = { warnings = { - type = "gcs" - destination = module.gcs.name - filter = "severity=WARNING" - grant = false + type = "gcs" + destination = module.gcs.name + filter = "severity=WARNING" + iam = false + include_children = true } info = { - type = "bigquery" - destination = module.dataset.id - filter = "severity=INFO" - grant = false + type = "bigquery" + destination = module.dataset.id + filter = "severity=INFO" + iam = false + include_children = true } notice = { - type = "pubsub" - destination = module.pubsub.id - filter = "severity=NOTICE" - grant = true + type = "pubsub" + destination = module.pubsub.id + filter = "severity=NOTICE" + iam = true + include_children = true } } logging_exclusions = { @@ -126,7 +129,7 @@ module "org" { | *iam_additive_members* | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | | *iam_audit_config* | Service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. | map(map(list(string))) | | {} | | *logging_exclusions* | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | -| *logging_sinks* | Logging sinks to create for this organization. | map(object({...})) | | {} | +| *logging_sinks* | Logging sinks to create for this organization. | map(object({...})) | | {} | | *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | | *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({...})) | | {} | diff --git a/modules/organization/main.tf b/modules/organization/main.tf index a764710c..1c115f24 100644 --- a/modules/organization/main.tf +++ b/modules/organization/main.tf @@ -53,7 +53,7 @@ locals { type => { for name, sink in local.logging_sinks : name => sink - if sink.grant && sink.type == type + if sink.iam && sink.type == type } } } @@ -221,9 +221,10 @@ resource "google_logging_organization_sink" "sink" { for_each = local.logging_sinks name = each.key #description = "${each.key} (Terraform-managed)" - org_id = local.organization_id_numeric - destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" - filter = each.value.filter + org_id = local.organization_id_numeric + destination = "${local.sink_type_destination[each.value.type]}/${each.value.destination}" + filter = each.value.filter + include_children = each.value.include_children } resource "google_storage_bucket_iam_binding" "gcs-sinks-binding" { @@ -249,13 +250,6 @@ resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" { members = [google_logging_organization_sink.sink[each.key].writer_identity] } -# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" { -# for_each = local.sink_grants["gcs"] -# bucket = each.value.destination -# role = "roles/storage.objectCreator" -# members = [google_logging_organization_sink.sink[each.key].writer_identity] -# } - resource "google_logging_organization_exclusion" "logging-exclusion" { for_each = coalesce(var.logging_exclusions, {}) name = each.key diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf index 5c426cf8..976bfeb7 100644 --- a/modules/organization/variables.tf +++ b/modules/organization/variables.tf @@ -102,10 +102,11 @@ variable "firewall_policy_attachments" { variable "logging_sinks" { description = "Logging sinks to create for this organization." type = map(object({ - destination = string - type = string - filter = string - grant = bool + destination = string + type = string + filter = string + iam = bool + include_children = bool })) default = {} } diff --git a/modules/project/README.md b/modules/project/README.md index f8d238d1..402cee0b 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -115,19 +115,19 @@ module "project-host" { type = "gcs" destination = module.gcs.name filter = "severity=WARNING" - grant = false + iam = false } info = { type = "bigquery" destination = module.dataset.id filter = "severity=INFO" - grant = false + iam = false } notice = { type = "pubsub" destination = module.pubsub.id filter = "severity=NOTICE" - grant = true + iam = true } } logging_exclusions = { @@ -153,7 +153,7 @@ module "project-host" { | *labels* | Resource labels. | map(string) | | {} | | *lien_reason* | If non-empty, creates a project lien with this description. | string | | | | *logging_exclusions* | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | -| *logging_sinks* | Logging sinks to create for this project. | map(object({...})) | | {} | +| *logging_sinks* | Logging sinks to create for this project. | map(object({...})) | | {} | | *oslogin* | Enable OS Login. | bool | | false | | *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | | *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | diff --git a/modules/project/main.tf b/modules/project/main.tf index 0606fa24..0a4d9b72 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -50,7 +50,7 @@ locals { type => { for name, sink in local.logging_sinks : name => sink - if sink.grant && sink.type == type + if sink.iam && sink.type == type } } } @@ -291,13 +291,6 @@ resource "google_pubsub_topic_iam_binding" "pubsub-sinks-binding" { members = [google_logging_project_sink.sink[each.key].writer_identity] } -# resource "google_storage_bucket_iam_binding" "gcs-sinks-bindings" { -# for_each = local.sink_grants["gcs"] -# bucket = each.value.destination -# role = "roles/storage.objectCreator" -# members = [google_logging_project_sink.sink[each.key].writer_identity] -# } - resource "google_logging_project_exclusion" "logging-exclusion" { for_each = coalesce(var.logging_exclusions, {}) name = each.key diff --git a/modules/project/variables.tf b/modules/project/variables.tf index d8e06a9e..b180ebdc 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -172,7 +172,7 @@ variable "logging_sinks" { destination = string type = string filter = string - grant = bool + iam = bool })) default = {} } diff --git a/tests/conftest.py b/tests/conftest.py index 7429269e..c50d8ec2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,12 +26,12 @@ BASEDIR = os.path.dirname(os.path.dirname(__file__)) def _plan_runner(): "Returns a function to run Terraform plan on a fixture." - def run_plan(fixture_path, targets=None, **tf_vars): + def run_plan(fixture_path, targets=None, refresh=True, **tf_vars): "Runs Terraform plan and returns parsed output." tf = tftest.TerraformTest(fixture_path, BASEDIR, os.environ.get('TERRAFORM', 'terraform')) tf.setup() - return tf.plan(output=True, tf_vars=tf_vars, targets=targets) + return tf.plan(output=True, refresh=refresh, tf_vars=tf_vars, targets=targets) return run_plan @@ -54,9 +54,9 @@ def plan_runner(_plan_runner): def e2e_plan_runner(_plan_runner): "Returns a function to run Terraform plan on an end-to-end fixture." - def run_plan(fixture_path, targets=None, **tf_vars): + def run_plan(fixture_path, targets=None, refresh=True, **tf_vars): "Runs Terraform plan on an end-to-end module using defaults, returns data." - plan = _plan_runner(fixture_path, targets=targets, **tf_vars) + plan = _plan_runner(fixture_path, targets=targets, refresh=refresh, **tf_vars) # skip the fixture root_module = plan.root_module['child_modules'][0] modules = dict((mod['address'], mod['resources']) diff --git a/tests/foundations/business_units/test_plan.py b/tests/foundations/business_units/test_plan.py index 0e400b36..de1a2afd 100644 --- a/tests/foundations/business_units/test_plan.py +++ b/tests/foundations/business_units/test_plan.py @@ -23,5 +23,5 @@ FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner(FIXTURES_DIR) - assert len(modules) == 9 - assert len(resources) == 84 + assert len(modules) == 8 + assert len(resources) == 82 diff --git a/tests/foundations/environments/test_plan.py b/tests/foundations/environments/test_plan.py index ef3f7d79..4e65fc3e 100644 --- a/tests/foundations/environments/test_plan.py +++ b/tests/foundations/environments/test_plan.py @@ -22,7 +22,7 @@ FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') def test_folder_roles(e2e_plan_runner): "Test folder roles." - modules, _ = e2e_plan_runner(FIXTURES_DIR) + modules, _ = e2e_plan_runner(FIXTURES_DIR, refresh=False) for env in ['test', 'prod']: resources = modules[f'module.test.module.environment-folders["{env}"]'] folders = [r for r in resources if r['type'] == 'google_folder'] @@ -41,7 +41,7 @@ def test_org_roles(e2e_plan_runner): 'organization_id': 'organizations/123', 'iam_xpn_config': '{grant = true, target_org = true}' } - modules, _ = e2e_plan_runner(FIXTURES_DIR, **tf_vars) + modules, _ = e2e_plan_runner(FIXTURES_DIR, refresh=False, **tf_vars) for env in ['test', 'prod']: resources = modules[f'module.test.module.environment-folders["{env}"]'] folder_bindings = [r['index'] diff --git a/tests/modules/folder/fixture/main.tf b/tests/modules/folder/fixture/main.tf index 5607b366..54b7c5fb 100644 --- a/tests/modules/folder/fixture/main.tf +++ b/tests/modules/folder/fixture/main.tf @@ -23,4 +23,6 @@ module "test" { policy_list = var.policy_list firewall_policies = var.firewall_policies firewall_policy_attachments = var.firewall_policy_attachments + logging_sinks = var.logging_sinks + logging_exclusions = var.logging_exclusions } diff --git a/tests/modules/folder/fixture/variables.tf b/tests/modules/folder/fixture/variables.tf index 908b2cb9..b25bf34c 100644 --- a/tests/modules/folder/fixture/variables.tf +++ b/tests/modules/folder/fixture/variables.tf @@ -53,3 +53,19 @@ variable "firewall_policy_attachments" { type = map(string) default = {} } + +variable "logging_sinks" { + type = map(object({ + destination = string + type = string + filter = string + iam = bool + include_children = bool + })) + default = {} +} + +variable "logging_exclusions" { + type = map(string) + default = {} +} diff --git a/tests/modules/folder/test_plan_logging.py b/tests/modules/folder/test_plan_logging.py new file mode 100644 index 00000000..65e40b26 --- /dev/null +++ b/tests/modules/folder/test_plan_logging.py @@ -0,0 +1,115 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + +from collections import Counter + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + + +def test_sinks(plan_runner): + "Test folder-level sinks." + logging_sinks = """ { + warning = { + type = "gcs" + destination = "mybucket" + filter = "severity=WARNING" + iam = true + include_children = true + } + info = { + type = "bigquery" + destination = "projects/myproject/datasets/mydataset" + filter = "severity=INFO" + iam = true + include_children = true + } + notice = { + type = "pubsub" + destination = "projects/myproject/topics/mytopic" + filter = "severity=NOTICE" + iam = true + include_children = false + } + } + """ + _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) + assert len(resources) == 7 + + resource_types = Counter([r['type'] for r in resources]) + assert resource_types == { + 'google_bigquery_dataset_iam_binding': 1, + 'google_folder': 1, + 'google_logging_folder_sink': 3, + 'google_pubsub_topic_iam_binding': 1, + 'google_storage_bucket_iam_binding': 1 + } + + sinks = [r for r in resources + if r['type'] == 'google_logging_folder_sink'] + assert sorted([r['index'] for r in sinks]) == [ + 'info', + 'notice', + 'warning', + ] + values = [(r['index'], r['values']['filter'], r['values']['destination'], + r['values']['include_children']) + for r in sinks] + assert sorted(values) == [ + ('info', + 'severity=INFO', + 'bigquery.googleapis.com/projects/myproject/datasets/mydataset', + True), + ('notice', + 'severity=NOTICE', + 'pubsub.googleapis.com/projects/myproject/topics/mytopic', + False), + ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket', True)] + + bindings = [r for r in resources + if 'binding' in r['type']] + values = [(r['index'], r['type'], r['values']['role']) + for r in bindings] + assert sorted(values) == [ + ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'), + ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'), + ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator') + ] + + +def test_exclusions(plan_runner): + "Test folder-level logging exclusions." + logging_exclusions = ( + '{' + 'exclusion1 = "resource.type=gce_instance", ' + 'exclusion2 = "severity=NOTICE", ' + '}' + ) + _, resources = plan_runner(FIXTURES_DIR, + logging_exclusions=logging_exclusions) + assert len(resources) == 3 + exclusions = [r for r in resources + if r['type'] == 'google_logging_folder_exclusion'] + assert sorted([r['index'] for r in exclusions]) == [ + 'exclusion1', + 'exclusion2', + ] + values = [(r['index'], r['values']['filter']) for r in exclusions] + assert sorted(values) == [ + ('exclusion1', 'resource.type=gce_instance'), + ('exclusion2', 'severity=NOTICE') + ] diff --git a/tests/modules/folder/test_plan_org_policies.py b/tests/modules/folder/test_plan_org_policies.py index 09d83cf5..35433cc1 100644 --- a/tests/modules/folder/test_plan_org_policies.py +++ b/tests/modules/folder/test_plan_org_policies.py @@ -20,8 +20,8 @@ import pytest FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') -def test_policy_boolean(plan_runner): - "Test boolean folder policy." +def test_sink(plan_runner): + "Test folder-level sink." policy_boolean = '{policy-a = true, policy-b = false, policy-c = null}' _, resources = plan_runner(FIXTURES_DIR, policy_boolean=policy_boolean) @@ -46,8 +46,8 @@ def test_policy_boolean(plan_runner): ] -def test_policy_list(plan_runner): - "Test list org policy." +def test_exclussions(plan_runner): + "Test folder-level logging exclusions." policy_list = ( '{' 'policy-a = {inherit_from_parent = true, suggested_value = null, status = true, values = []}, ' diff --git a/tests/modules/organization/fixture/main.tf b/tests/modules/organization/fixture/main.tf index 28bbe270..db7264f8 100644 --- a/tests/modules/organization/fixture/main.tf +++ b/tests/modules/organization/fixture/main.tf @@ -26,4 +26,6 @@ module "test" { policy_list = var.policy_list firewall_policies = var.firewall_policies firewall_policy_attachments = var.firewall_policy_attachments + logging_sinks = var.logging_sinks + logging_exclusions = var.logging_exclusions } diff --git a/tests/modules/organization/fixture/variables.tf b/tests/modules/organization/fixture/variables.tf index 7fe88394..9f54a034 100644 --- a/tests/modules/organization/fixture/variables.tf +++ b/tests/modules/organization/fixture/variables.tf @@ -73,3 +73,19 @@ variable "firewall_policy_attachments" { type = map(string) default = {} } + +variable "logging_sinks" { + type = map(object({ + destination = string + type = string + filter = string + iam = bool + include_children = bool + })) + default = {} +} + +variable "logging_exclusions" { + type = map(string) + default = {} +} diff --git a/tests/modules/organization/test_plan_logging.py b/tests/modules/organization/test_plan_logging.py new file mode 100644 index 00000000..5a7257fb --- /dev/null +++ b/tests/modules/organization/test_plan_logging.py @@ -0,0 +1,114 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + +from collections import Counter + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + + +def test_sinks(plan_runner): + "Test folder-level sinks." + logging_sinks = """ { + warning = { + type = "gcs" + destination = "mybucket" + filter = "severity=WARNING" + iam = true + include_children = true + } + info = { + type = "bigquery" + destination = "projects/myproject/datasets/mydataset" + filter = "severity=INFO" + iam = true + include_children = true + } + notice = { + type = "pubsub" + destination = "projects/myproject/topics/mytopic" + filter = "severity=NOTICE" + iam = true + include_children = false + } + } + """ + _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) + assert len(resources) == 6 + + resource_types = Counter([r['type'] for r in resources]) + assert resource_types == { + 'google_bigquery_dataset_iam_binding': 1, + 'google_logging_organization_sink': 3, + 'google_pubsub_topic_iam_binding': 1, + 'google_storage_bucket_iam_binding': 1 + } + + sinks = [r for r in resources + if r['type'] == 'google_logging_organization_sink'] + assert sorted([r['index'] for r in sinks]) == [ + 'info', + 'notice', + 'warning', + ] + values = [(r['index'], r['values']['filter'], r['values']['destination'], + r['values']['include_children']) + for r in sinks] + assert sorted(values) == [ + ('info', + 'severity=INFO', + 'bigquery.googleapis.com/projects/myproject/datasets/mydataset', + True), + ('notice', + 'severity=NOTICE', + 'pubsub.googleapis.com/projects/myproject/topics/mytopic', + False), + ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket', True)] + + bindings = [r for r in resources + if 'binding' in r['type']] + values = [(r['index'], r['type'], r['values']['role']) + for r in bindings] + assert sorted(values) == [ + ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'), + ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'), + ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator') + ] + + +def test_exclusions(plan_runner): + "Test folder-level logging exclusions." + logging_exclusions = ( + '{' + 'exclusion1 = "resource.type=gce_instance", ' + 'exclusion2 = "severity=NOTICE", ' + '}' + ) + _, resources = plan_runner(FIXTURES_DIR, + logging_exclusions=logging_exclusions) + assert len(resources) == 2 + exclusions = [r for r in resources + if r['type'] == 'google_logging_organization_exclusion'] + assert sorted([r['index'] for r in exclusions]) == [ + 'exclusion1', + 'exclusion2', + ] + values = [(r['index'], r['values']['filter']) for r in exclusions] + assert sorted(values) == [ + ('exclusion1', 'resource.type=gce_instance'), + ('exclusion2', 'severity=NOTICE') + ] diff --git a/tests/modules/project/fixture/main.tf b/tests/modules/project/fixture/main.tf index acec2c3f..2c2197a7 100644 --- a/tests/modules/project/fixture/main.tf +++ b/tests/modules/project/fixture/main.tf @@ -33,4 +33,6 @@ module "test" { policy_list = var.policy_list prefix = var.prefix services = var.services + logging_sinks = var.logging_sinks + logging_exclusions = var.logging_exclusions } diff --git a/tests/modules/project/fixture/variables.tf b/tests/modules/project/fixture/variables.tf index 3b759ddb..2de1e779 100644 --- a/tests/modules/project/fixture/variables.tf +++ b/tests/modules/project/fixture/variables.tf @@ -93,3 +93,18 @@ variable "services" { type = list(string) default = [] } + +variable "logging_sinks" { + type = map(object({ + destination = string + type = string + filter = string + iam = bool + })) + default = {} +} + +variable "logging_exclusions" { + type = map(string) + default = {} +} diff --git a/tests/modules/project/test_plan_logging.py b/tests/modules/project/test_plan_logging.py new file mode 100644 index 00000000..fb6d8ed6 --- /dev/null +++ b/tests/modules/project/test_plan_logging.py @@ -0,0 +1,109 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + +from collections import Counter + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + + +def test_sinks(plan_runner): + "Test folder-level sinks." + logging_sinks = """ { + warning = { + type = "gcs" + destination = "mybucket" + filter = "severity=WARNING" + iam = true + } + info = { + type = "bigquery" + destination = "projects/myproject/datasets/mydataset" + filter = "severity=INFO" + iam = true + } + notice = { + type = "pubsub" + destination = "projects/myproject/topics/mytopic" + filter = "severity=NOTICE" + iam = true + } + } + """ + _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) + assert len(resources) == 7 + + resource_types = Counter([r['type'] for r in resources]) + assert resource_types == { + 'google_bigquery_dataset_iam_binding': 1, + 'google_logging_project_sink': 3, + 'google_project': 1, + 'google_pubsub_topic_iam_binding': 1, + 'google_storage_bucket_iam_binding': 1 + } + + sinks = [r for r in resources + if r['type'] == 'google_logging_project_sink'] + assert sorted([r['index'] for r in sinks]) == [ + 'info', + 'notice', + 'warning', + ] + values = [(r['index'], r['values']['filter'], r['values']['destination']) + for r in sinks] + assert sorted(values) == [ + ('info', + 'severity=INFO', + 'bigquery.googleapis.com/projects/myproject/datasets/mydataset'), + ('notice', + 'severity=NOTICE', + 'pubsub.googleapis.com/projects/myproject/topics/mytopic'), + ('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket')] + + bindings = [r for r in resources + if 'binding' in r['type']] + values = [(r['index'], r['type'], r['values']['role']) + for r in bindings] + assert sorted(values) == [ + ('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'), + ('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'), + ('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator') + ] + + +def test_exclusions(plan_runner): + "Test folder-level logging exclusions." + logging_exclusions = ( + '{' + 'exclusion1 = "resource.type=gce_instance", ' + 'exclusion2 = "severity=NOTICE", ' + '}' + ) + _, resources = plan_runner(FIXTURES_DIR, + logging_exclusions=logging_exclusions) + assert len(resources) == 3 + exclusions = [r for r in resources + if r['type'] == 'google_logging_project_exclusion'] + assert sorted([r['index'] for r in exclusions]) == [ + 'exclusion1', + 'exclusion2', + ] + values = [(r['index'], r['values']['filter']) for r in exclusions] + assert sorted(values) == [ + ('exclusion1', 'resource.type=gce_instance'), + ('exclusion2', 'severity=NOTICE') + ]