From 841459a059703d9e1083c164e1e1a0afc1d2a349 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 14 Aug 2023 07:52:36 +0200 Subject: [PATCH 01/46] Fix factory rules key in net firewall policy module (#1587) * fix factory rules key in net firewall policy * fix test --- modules/net-firewall-policy/factory.tf | 4 ++-- tests/modules/net_firewall_policy/examples/factory.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/net-firewall-policy/factory.tf b/modules/net-firewall-policy/factory.tf index cb567338..4a9c8558 100644 --- a/modules/net-firewall-policy/factory.tf +++ b/modules/net-firewall-policy/factory.tf @@ -25,7 +25,7 @@ locals { yamldecode(file(var.rules_factory_config.cidr_file_path)), {} ) factory_egress_rules = { - for k, v in local._factory_egress_rules : "ingress/${k}" => { + for k, v in local._factory_egress_rules : "egress/${k}" => { action = "deny" direction = "EGRESS" priority = v.priority @@ -68,7 +68,7 @@ locals { } } factory_ingress_rules = { - for k, v in local._factory_ingress_rules : "egress/${k}" => { + for k, v in local._factory_ingress_rules : "ingress/${k}" => { action = "allow" direction = "INGRESS" priority = v.priority diff --git a/tests/modules/net_firewall_policy/examples/factory.yaml b/tests/modules/net_firewall_policy/examples/factory.yaml index b3709fe6..216a945a 100644 --- a/tests/modules/net_firewall_policy/examples/factory.yaml +++ b/tests/modules/net_firewall_policy/examples/factory.yaml @@ -18,7 +18,7 @@ values: module.firewall-policy.google_compute_firewall_policy_association.hierarchical["test"]: attachment_target: folders/4567890123 name: test-1-test - module.firewall-policy.google_compute_firewall_policy_rule.hierarchical["egress/icmp"]: + module.firewall-policy.google_compute_firewall_policy_rule.hierarchical["ingress/icmp"]: action: allow direction: INGRESS disabled: false @@ -41,7 +41,7 @@ values: priority: 1000 target_resources: null target_service_accounts: null - module.firewall-policy.google_compute_firewall_policy_rule.hierarchical["ingress/smtp"]: + module.firewall-policy.google_compute_firewall_policy_rule.hierarchical["egress/smtp"]: action: deny direction: EGRESS disabled: false From ec56a86bbd745ddbfdc519857253e1a84d1d327c Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 14 Aug 2023 07:54:08 +0200 Subject: [PATCH 02/46] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ebc65e5..ef1e4e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. ### MODULES +- [[#1587](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1587)] **incompatible change:** Fix factory rules key in net firewall policy module ([ludoo](https://github.com/ludoo)) - [[#1578](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1578)] Fix: Instance level stateful disk config ([beardedsamwise](https://github.com/beardedsamwise)) - [[#1582](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1582)] feat(modules/cloud-run): add gen2 exec env support ([LiuVII](https://github.com/LiuVII)) From f9509ad6b70ff0e14a0ce46818455e3660f66ede Mon Sep 17 00:00:00 2001 From: erabusi <72961506+erabusi@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:07:03 +0530 Subject: [PATCH 03/46] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20(modules/cloudsql-?= =?UTF-8?q?instance):=20enable=20require=5Fssl=20cert=20support=20(#1588)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudsql-instance/README.md | 7 ++++--- modules/cloudsql-instance/main.tf | 1 + modules/cloudsql-instance/variables.tf | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/cloudsql-instance/README.md b/modules/cloudsql-instance/README.md index ecd26017..172a805e 100644 --- a/modules/cloudsql-instance/README.md +++ b/modules/cloudsql-instance/README.md @@ -193,7 +193,7 @@ module "db" { | [network](variables.tf#L130) | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | string | ✓ | | | [project_id](variables.tf#L151) | The ID of the project where this instances will be created. | string | ✓ | | | [region](variables.tf#L156) | Region of the primary instance. | string | ✓ | | -| [tier](variables.tf#L176) | The machine type to use for the instances. | string | ✓ | | +| [tier](variables.tf#L182) | The machine type to use for the instances. | string | ✓ | | | [allocated_ip_ranges](variables.tf#L17) | (Optional)The name of the allocated ip range for the private ip CloudSQL instance. For example: \"google-managed-services-default\". If set, the instance ip will be created in the allocated range. The range name must comply with RFC 1035. Specifically, the name must be 1-63 characters long and match the regular expression a-z?. | object({…}) | | {} | | [authorized_networks](variables.tf#L26) | Map of NAME=>CIDR_RANGE to allow to connect to the database(s). | map(string) | | null | | [availability_type](variables.tf#L32) | Availability type for the primary replica. Either `ZONAL` or `REGIONAL`. | string | | "ZONAL" | @@ -210,8 +210,9 @@ module "db" { | [postgres_client_certificates](variables.tf#L135) | Map of cert keys connect to the application(s) using public IP. | list(string) | | null | | [prefix](variables.tf#L141) | Optional prefix used to generate instance names. | string | | null | | [replicas](variables.tf#L161) | Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation. | map(object({…})) | | {} | -| [root_password](variables.tf#L170) | Root password of the Cloud SQL instance. Required for MS SQL Server. | string | | null | -| [users](variables.tf#L181) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | map(string) | | null | +| [require_ssl](variables.tf#L170) | Enable SSL connections only. | bool | | null | +| [root_password](variables.tf#L176) | Root password of the Cloud SQL instance. Required for MS SQL Server. | string | | null | +| [users](variables.tf#L187) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | map(string) | | null | ## Outputs diff --git a/modules/cloudsql-instance/main.tf b/modules/cloudsql-instance/main.tf index 67111a7b..5c40b212 100644 --- a/modules/cloudsql-instance/main.tf +++ b/modules/cloudsql-instance/main.tf @@ -64,6 +64,7 @@ resource "google_sql_database_instance" "primary" { ipv4_enabled = var.ipv4_enabled private_network = var.network allocated_ip_range = var.allocated_ip_ranges.primary + require_ssl = var.require_ssl dynamic "authorized_networks" { for_each = var.authorized_networks != null ? var.authorized_networks : {} iterator = network diff --git a/modules/cloudsql-instance/variables.tf b/modules/cloudsql-instance/variables.tf index 8b3e0fff..03e0d6df 100644 --- a/modules/cloudsql-instance/variables.tf +++ b/modules/cloudsql-instance/variables.tf @@ -167,6 +167,12 @@ variable "replicas" { default = {} } +variable "require_ssl" { + description = "Enable SSL connections only." + type = bool + default = null +} + variable "root_password" { description = "Root password of the Cloud SQL instance. Required for MS SQL Server." type = string From a509756f1b16ae0029736b898cb8e7d5e3b3a342 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Mon, 14 Aug 2023 11:48:27 +0200 Subject: [PATCH 04/46] GCVE module first release --- modules/gcve-private-cloud/README.md | 97 +++++++++++++++++++ modules/gcve-private-cloud/main.tf | 74 ++++++++++++++ modules/gcve-private-cloud/output.tf | 71 ++++++++++++++ modules/gcve-private-cloud/variables.tf | 92 ++++++++++++++++++ modules/gcve-private-cloud/versions.tf | 29 ++++++ .../gcve_private_cloud/examples/basic.yaml | 40 ++++++++ .../gcve_private_cloud/examples/custom.yaml | 42 ++++++++ 7 files changed, 445 insertions(+) create mode 100644 modules/gcve-private-cloud/README.md create mode 100644 modules/gcve-private-cloud/main.tf create mode 100644 modules/gcve-private-cloud/output.tf create mode 100644 modules/gcve-private-cloud/variables.tf create mode 100644 modules/gcve-private-cloud/versions.tf create mode 100644 tests/modules/gcve_private_cloud/examples/basic.yaml create mode 100644 tests/modules/gcve_private_cloud/examples/custom.yaml diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md new file mode 100644 index 00000000..8baad718 --- /dev/null +++ b/modules/gcve-private-cloud/README.md @@ -0,0 +1,97 @@ +# Google Cloud VMWare Engine Private Cloud Module + +This module implements the creation and management of a Google Cloud VMWare Engine Private Cloud with its management cluster. If configured, it also creates the vmware engine network or it can work with an existing one. The creation of the private connection with the user VPC requires the execution of the [Google SDK command](https://cloud.google.com/sdk/gcloud/reference/vmware/private-connections/create#--routing-mode) the module provides as an output. + +Be aware that the deployment of this module might requires up to 2 hours depending on the selected private cloud target zone. + +## TOC + + +- [TOC](#toc) +- [Limitations](#limitations) +- [Basic Private Cloud Creation](#basic-private-cloud-creation) +- [Private Cloud Creation with custom nodes and cores count](#private-cloud-creation-with-custom-nodes-and-cores-count) +- [Files](#files) +- [Variables](#variables) + + +## Limitations +At the moment this module doesn't support the following use cases: +- `Single node private cloud` +- `Stretched private cloud` + +## Basic Private Cloud Creation + +```hcl +module "gcve-pc" { + source = "./fabric/modules/gcve-private-cloud" + name = "gcve-pc" + project_id = "gcve-test-project" + zone = "asia-southeast1-a" + management_cidr = "192.168.0.0/24" + + private_connections = { + transit-conn1 = { + name = "transit-conn1", + network_self_link = "projects/test-prj-elia-01/global/networks/default", + peering = "servicenetworking-googleapis-com" + type = "PRIVATE_SERVICE_ACCESS", + routing_mode = "REGIONAL" + } + } +} +# tftest modules=1 resources=2 inventory=basic.yaml +``` +## Private Cloud Creation with custom nodes and cores count + +```hcl +module "gcve-pc" { + source = "./fabric/modules/gcve-private-cloud" + name = "gcve-pc" + project_id = "gcve-test-project" + zone = "asia-southeast1-a" + management_cidr = "192.168.0.0/24" + + management_cluster_config = { + node_type_id = "standard-72" + node_count = 6 + custom_core_count = 28 + } + + private_connections = { + transit-conn1 = { + name = "transit-conn1", + network_self_link = "projects/test-prj-elia-01/global/networks/default", + peering = "servicenetworking-googleapis-com" + type = "PRIVATE_SERVICE_ACCESS", + routing_mode = "REGIONAL" + } + } +} +# tftest modules=1 resources=2 inventory=custom.yaml +``` + + + +## Files + +| name | description | resources | +|---|---|---| +| [main.tf](./main.tf) | Module-level locals and resources. | google_vmwareengine_network · google_vmwareengine_private_cloud | +| [output.tf](./output.tf) | None | | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [management_cidr](variables.tf#L23) | vSphere/vSAN subnets CIDR range. | string | ✓ | | +| [name](variables.tf#L42) | Private cloud name. | string | ✓ | | +| [project_id](variables.tf#L74) | Project id. | string | ✓ | | +| [zone](variables.tf#L85) | Private cloud zone. | string | ✓ | | +| [description](variables.tf#L17) | Private cloud description. | string | | "Terraform-managed." | +| [management_cluster_config](variables.tf#L28) | Management cluster configuration. | object({…}) | | {…} | +| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [vmwareengine_network_create](variables.tf#L79) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | + diff --git a/modules/gcve-private-cloud/main.tf b/modules/gcve-private-cloud/main.tf new file mode 100644 index 00000000..3f184475 --- /dev/null +++ b/modules/gcve-private-cloud/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + region = join("-", slice(split("-", "${var.zone}"), 0, 2)) + vmwareengine_network = ( + var.vmwareengine_network_create + ? try(google_vmwareengine_network.private-cloud-network.0, null) + : try(data.google_vmwareengine_network.private-cloud-network.0, null) + ) + psa_peering = { + for k, v in data.google_compute_network_peering.psa_peering : k => slice(split("/", "${v.peer_network}"), 6, 7)[0] + } +} + +data "google_vmwareengine_network" "private-cloud-network" { + count = var.vmwareengine_network_create ? 0 : 1 + provider = google-beta + project = var.project_id + name = "${local.region}-default" + location = local.region +} + + +data "google_compute_network_peering" "psa_peering" { + for_each = var.private_connections + name = each.value.peering + network = each.value.network_self_link +} + +resource "google_vmwareengine_private_cloud" "private-cloud" { + provider = google-beta + project = var.project_id + location = var.zone + name = var.name + description = var.description + + network_config { + management_cidr = var.management_cidr + vmware_engine_network = local.vmwareengine_network.id + } + + management_cluster { + cluster_id = "${var.name}-mgmt-cluster" + node_type_configs { + node_type_id = var.management_cluster_config.node_type_id + node_count = var.management_cluster_config.node_count + custom_core_count = var.management_cluster_config.custom_core_count + } + } +} + +resource "google_vmwareengine_network" "private-cloud-network" { + count = var.vmwareengine_network_create ? 1 : 0 + provider = google-beta + project = var.project_id + name = "${local.region}-default" + location = local.region + type = "LEGACY" + description = "Private cloud ${var.name} network." +} diff --git a/modules/gcve-private-cloud/output.tf b/modules/gcve-private-cloud/output.tf new file mode 100644 index 00000000..f1af67ec --- /dev/null +++ b/modules/gcve-private-cloud/output.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "hcx" { + description = "Details about a HCX Cloud Manager appliance." + value = google_vmwareengine_private_cloud.private-cloud.hcx +} + +output "id" { + description = "ID of the private cloud" + value = google_vmwareengine_private_cloud.private-cloud.id +} + +output "management_cluster" { + description = "Details of the management cluster of the private cloud" + value = google_vmwareengine_private_cloud.private-cloud.management_cluster +} + +output "network_config" { + description = "Details about the network configuration of the private cloud" + value = google_vmwareengine_private_cloud.private-cloud.network_config +} + +output "nsx" { + description = "Details about a NSX Manager appliance." + value = google_vmwareengine_private_cloud.private-cloud.nsx +} + +output "private-cloud" { + description = "The private cloud resource" + value = google_vmwareengine_private_cloud.private-cloud +} + +output "vcenter" { + description = "Details about a vCenter Server management appliance." + value = google_vmwareengine_private_cloud.private-cloud.vcenter +} + +output "state" { + description = "Details about the state of the private cloud" + value = google_vmwareengine_private_cloud.private-cloud.state +} + +output "private_connections_setup" { + description = "Cloud SDK commands for the private connections manual setup." + value = { + for k, v in var.private_connections : k => < Date: Mon, 14 Aug 2023 11:54:50 +0200 Subject: [PATCH 05/46] Add new `iam_members` variable to IAM additive module interfaces (#1589) * resource management modules * data catalog policy * dataproc * service account * kms * net-vpc * source repository * dataplex datascan * service account module variable order --- modules/data-catalog-policy-tag/README.md | 28 ++++-- modules/data-catalog-policy-tag/iam.tf | 10 +- modules/data-catalog-policy-tag/variables.tf | 10 ++ modules/dataplex-datascan/README.md | 53 ++++++---- modules/dataplex-datascan/iam.tf | 9 ++ modules/dataplex-datascan/variables.tf | 10 ++ modules/dataproc/README.md | 42 +++++--- modules/dataproc/iam.tf | 8 ++ modules/dataproc/variables.tf | 10 ++ modules/folder/README.md | 33 ++++--- modules/folder/iam.tf | 7 ++ modules/folder/variables.tf | 10 ++ modules/iam-service-account/README.md | 21 ++-- modules/iam-service-account/iam.tf | 7 ++ modules/iam-service-account/variables.tf | 10 ++ modules/kms/README.md | 41 +++++--- modules/kms/iam.tf | 97 +++++++++++++++++++ modules/kms/main.tf | 65 ------------- modules/kms/variables.tf | 23 ++++- modules/net-vpc/README.md | 18 +++- modules/net-vpc/subnets.tf | 9 ++ modules/net-vpc/variables.tf | 11 +++ modules/organization/README.md | 37 ++++--- modules/organization/iam.tf | 7 ++ modules/organization/variables.tf | 10 ++ modules/project/README.md | 93 ++++++++++++------ modules/project/iam.tf | 11 +++ modules/project/variables.tf | 10 ++ modules/source-repository/README.md | 26 +++-- modules/source-repository/iam.tf | 8 ++ modules/source-repository/variables.tf | 10 ++ .../examples/datascan_iam.yaml | 13 ++- tests/modules/folder/examples/iam.yaml | 11 ++- tests/modules/kms/examples/basic.yaml | 22 ++++- .../modules/net_vpc/examples/subnet-iam.yaml | 69 +++++++++++-- .../modules/organization/examples/basic.yaml | 36 ++++++- .../modules/project/examples/iam-members.yaml | 48 +++++++++ .../source_repository/examples/simple.yaml | 12 +++ 38 files changed, 748 insertions(+), 207 deletions(-) create mode 100644 modules/kms/iam.tf create mode 100644 tests/modules/project/examples/iam-members.yaml diff --git a/modules/data-catalog-policy-tag/README.md b/modules/data-catalog-policy-tag/README.md index 99f2c08c..6f53fcbd 100644 --- a/modules/data-catalog-policy-tag/README.md +++ b/modules/data-catalog-policy-tag/README.md @@ -4,6 +4,15 @@ This module simplifies the creation of [Data Catalog](https://cloud.google.com/d Note: Data Catalog is still in beta, hence this module currently uses the beta provider. + +- [Examples](#examples) + - [Simple Taxonomy with policy tags](#simple-taxonomy-with-policy-tags) + - [Taxonomy with IAM binding](#taxonomy-with-iam-binding) +- [Variables](#variables) +- [Outputs](#outputs) +- [TODO](#todo) + + ## Examples ### Simple Taxonomy with policy tags @@ -43,25 +52,32 @@ module "cmn-dc" { iam = { "roles/datacatalog.categoryAdmin" = ["group:GROUP_NAME@example.com"] } + iam_members = { + am1-admin = { + member = "user:am1@example.com" + role = "roles/datacatalog.categoryAdmin" + } + } } -# tftest modules=1 resources=6 +# tftest modules=1 resources=7 ``` ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L59) | Name of this taxonomy. | string | ✓ | | -| [project_id](variables.tf#L74) | GCP project id. | | ✓ | | +| [name](variables.tf#L69) | Name of this taxonomy. | string | ✓ | | +| [project_id](variables.tf#L84) | GCP project id. | | ✓ | | | [activated_policy_types](variables.tf#L17) | A list of policy types that are activated for this taxonomy. | list(string) | | ["FINE_GRAINED_ACCESS_CONTROL"] | | [description](variables.tf#L23) | Description of this taxonomy. | string | | "Taxonomy - Terraform managed" | | [group_iam](variables.tf#L29) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L35) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L41) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L47) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [location](variables.tf#L53) | Data Catalog Taxonomy location. | string | | "eu" | -| [prefix](variables.tf#L64) | Optional prefix used to generate project id and name. | string | | null | -| [tags](variables.tf#L78) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | +| [iam_members](variables.tf#L53) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [location](variables.tf#L63) | Data Catalog Taxonomy location. | string | | "eu" | +| [prefix](variables.tf#L74) | Optional prefix used to generate project id and name. | string | | null | +| [tags](variables.tf#L88) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | ## Outputs diff --git a/modules/data-catalog-policy-tag/iam.tf b/modules/data-catalog-policy-tag/iam.tf index d682e6e8..6f79aebf 100644 --- a/modules/data-catalog-policy-tag/iam.tf +++ b/modules/data-catalog-policy-tag/iam.tf @@ -58,9 +58,9 @@ locals { resource "google_data_catalog_taxonomy_iam_binding" "authoritative" { provider = google-beta for_each = local.iam + taxonomy = google_data_catalog_taxonomy.default.id role = each.key members = each.value - taxonomy = google_data_catalog_taxonomy.default.id } resource "google_data_catalog_taxonomy_iam_member" "additive" { @@ -70,9 +70,16 @@ resource "google_data_catalog_taxonomy_iam_member" "additive" { ? local.iam_additive : {} ) + taxonomy = google_data_catalog_taxonomy.default.id role = each.value.role member = each.value.member +} + +resource "google_data_catalog_taxonomy_iam_member" "members" { + for_each = var.iam_members taxonomy = google_data_catalog_taxonomy.default.id + role = each.value.role + member = each.value.member } resource "google_data_catalog_policy_tag_iam_binding" "authoritative" { @@ -80,7 +87,6 @@ resource "google_data_catalog_policy_tag_iam_binding" "authoritative" { for_each = { for v in local.tags_iam : "${v.tag}.${v.role}" => v } - policy_tag = google_data_catalog_policy_tag.default[each.value.tag].name role = each.value.role members = each.value.members diff --git a/modules/data-catalog-policy-tag/variables.tf b/modules/data-catalog-policy-tag/variables.tf index 70ba24bf..c7c8eb07 100644 --- a/modules/data-catalog-policy-tag/variables.tf +++ b/modules/data-catalog-policy-tag/variables.tf @@ -50,6 +50,16 @@ variable "iam_additive_members" { default = {} } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "location" { description = "Data Catalog Taxonomy location." type = string diff --git a/modules/dataplex-datascan/README.md b/modules/dataplex-datascan/README.md index 96027a53..65436af6 100644 --- a/modules/dataplex-datascan/README.md +++ b/modules/dataplex-datascan/README.md @@ -2,9 +2,20 @@ This module manages the creation of Dataplex DataScan resources. + +- [Data Profiling](#data-profiling) +- [Data Quality](#data-quality) +- [Data Source](#data-source) +- [Execution Schedule](#execution-schedule) +- [IAM](#iam) +- [TODO](#todo) +- [Variables](#variables) +- [Outputs](#outputs) + + ## Data Profiling -This example shows how to create a Data Profiling scan. To create an Data Profiling scan, provide the `data_profile_spec` input arguments as documented in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec. +This example shows how to create a Data Profiling scan. To create an Data Profiling scan, provide the `data_profile_spec` input arguments as documented in . ```hcl module "dataplex-datascan" { @@ -30,9 +41,9 @@ module "dataplex-datascan" { ## Data Quality -To create an Data Quality scan, provide the `data_quality_spec` input arguments as documented in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. +To create an Data Quality scan, provide the `data_quality_spec` input arguments as documented in . -Documentation for the supported rule types and rule specifications can be found in https://cloud.example.com/dataplex/docs/reference/rest/v1/DataQualityRule. +Documentation for the supported rule types and rule specifications can be found in . This example shows how to create a Data Quality scan. @@ -304,6 +315,7 @@ rules: The input variable 'data' is required to create a DataScan. This value is immutable. Once it is set, you cannot change the DataScan to another source. The input variable 'data' should be an object containing a single key-value pair that can be one of: + * `entity`: The Dataplex entity that represents the data source (e.g. BigQuery table) for DataScan, of the form: `projects/{project_number}/locations/{locationId}/lakes/{lakeId}/zones/{zoneId}/entities/{entityId}`. * `resource`: The service-qualified full resource name of the cloud resource for a DataScan job to scan against. The field could be: BigQuery table of type "TABLE" for DataProfileScan/DataQualityScan format, e.g: `//bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID`. @@ -368,17 +380,17 @@ module "dataplex-datascan" { ## IAM -There are three mutually exclusive ways of managing IAM in this module +IAM is managed via several variables that implement different levels of control: -- non-authoritative via the `iam_additive` and `iam_additive_members` variables, where bindings created outside this module will coexist with those managed here -- authoritative via the `group_iam` and `iam` variables, where bindings created outside this module (eg in the console) will be removed at each `terraform apply` cycle if the same role is also managed here -- authoritative policy via the `iam_policy` variable, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role +* `group_iam` and `iam` configure authoritative bindings that manage individual roles exclusively, mapping to the [`google_project_iam_binding`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_binding) resource +* `iam_additive`, `iam_additive_members` and `iam_members` configure additive bindings that only manage individual role/member pairs, mapping to the [`google_project_iam_member`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_member) resource +* `iam_policy` which controls the entire IAM policy for the project, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role The authoritative and additive approaches can be used together, provided different roles are managed by each. The IAM policy is incompatible with the other approaches, and must be used with extreme care. -Some care must also be taken with the `group_iam` variable (and in some situations with the additive variables) to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. +Some care must also be taken with the `group_iam` and `iam_additive_*` variables to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. For additive roles `iam_members` ensures that no dynamic values are used in the internal loop. -An example is provided beow for using `group_iam` and `iam` variables. +An example is provided below for using some of these variables. ```hcl module "dataplex-datascan" { @@ -404,8 +416,14 @@ module "dataplex-datascan" { "roles/dataplex.dataScanViewer" ] } + iam_members = { + am1-viewer = { + member = "user:am1@example.com" + role = "roles/dataplex.dataScanViewer" + } + } } -# tftest modules=1 resources=4 inventory=datascan_iam.yaml +# tftest modules=1 resources=5 inventory=datascan_iam.yaml ``` ## TODO @@ -415,9 +433,9 @@ module "dataplex-datascan" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [data](variables.tf#L17) | The data source for DataScan. The source can be either a Dataplex `entity` or a BigQuery `resource`. | object({…}) | ✓ | | -| [name](variables.tf#L146) | Name of Dataplex Scan. | string | ✓ | | -| [project_id](variables.tf#L157) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | -| [region](variables.tf#L162) | Region for the Dataplex DataScan. | string | ✓ | | +| [name](variables.tf#L156) | Name of Dataplex Scan. | string | ✓ | | +| [project_id](variables.tf#L167) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | +| [region](variables.tf#L172) | Region for the Dataplex DataScan. | string | ✓ | | | [data_profile_spec](variables.tf#L29) | DataProfileScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec. | object({…}) | | null | | [data_quality_spec](variables.tf#L38) | DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | | [data_quality_spec_file](variables.tf#L80) | Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | @@ -427,10 +445,11 @@ module "dataplex-datascan" { | [iam](variables.tf#L107) | Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L114) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L121) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_policy](variables.tf#L127) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [incremental_field](variables.tf#L133) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | -| [labels](variables.tf#L139) | Resource labels. | map(string) | | {} | -| [prefix](variables.tf#L151) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | +| [iam_members](variables.tf#L127) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_policy](variables.tf#L137) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | +| [incremental_field](variables.tf#L143) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | +| [labels](variables.tf#L149) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L161) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | ## Outputs diff --git a/modules/dataplex-datascan/iam.tf b/modules/dataplex-datascan/iam.tf index 62eb2fd9..e1a7c057 100644 --- a/modules/dataplex-datascan/iam.tf +++ b/modules/dataplex-datascan/iam.tf @@ -69,6 +69,15 @@ resource "google_dataplex_datascan_iam_member" "additive" { member = each.value.member } +resource "google_dataplex_datascan_iam_member" "members" { + for_each = var.iam_members + project = google_dataplex_datascan.datascan.project + location = google_dataplex_datascan.datascan.location + data_scan_id = google_dataplex_datascan.datascan.data_scan_id + role = each.value.role + member = each.value.member +} + resource "google_dataplex_datascan_iam_policy" "authoritative_for_resource" { count = var.iam_policy != null ? 1 : 0 project = google_dataplex_datascan.datascan.project diff --git a/modules/dataplex-datascan/variables.tf b/modules/dataplex-datascan/variables.tf index 9d167840..47ca7332 100644 --- a/modules/dataplex-datascan/variables.tf +++ b/modules/dataplex-datascan/variables.tf @@ -124,6 +124,16 @@ variable "iam_additive_members" { default = {} } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "iam_policy" { description = "IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution." type = map(list(string)) diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index f848db57..3e454e50 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -2,6 +2,19 @@ This module Manages a Google Cloud [Dataproc](https://cloud.google.com/dataproc) cluster resource, including IAM. + +- [TODO](#todo) +- [Examples](#examples) + - [Simple](#simple) + - [Cluster configuration](#cluster-configuration) + - [Cluster with CMEK encryption](#cluster-with-cmek-encryption) +- [IAM Examples](#iam-examples) + - [Authoritative IAM](#authoritative-iam) + - [Additive IAM](#additive-iam) +- [Variables](#variables) +- [Outputs](#outputs) + + ## TODO - [ ] Add support for Cloud Dataproc [autoscaling policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/dataproc_autoscaling_policy_iam). @@ -79,8 +92,10 @@ module "processing-dp-cluster" { IAM is managed via several variables that implement different levels of control: -- `group_iam` and `iam` configure authoritative bindings that manage individual roles exclusively, mapping to the `google_dataproc_cluster_iam_binding` resource -- `iam_additive` configure additive bindings that only manage individual role/member pairs, mapping to the `google_dataproc_cluster_iam_member` resource +IAM is managed via several variables that implement different levels of control: + +- `group_iam` and `iam` configure authoritative bindings that manage individual roles exclusively, mapping to the [`google_project_iam_binding`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_binding) resource +- `iam_additive` and `iam_members` configures additive bindings that only manage individual role/member pairs, mapping to the [`google_project_iam_member`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_member) resource ### Authoritative IAM @@ -136,25 +151,31 @@ module "processing-dp-cluster" { "serviceAccount:service-account@PROJECT_ID.iam.gserviceaccount.com" ] } + iam_members = { + am1-viewer = { + member = "user:am1@example.com" + role = "roles/dataproc.viewer" + } + } } -# tftest modules=1 resources=2 +# tftest modules=1 resources=3 ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L212) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L227) | Project ID. | string | ✓ | | -| [region](variables.tf#L232) | Dataproc region. | string | ✓ | | +| [name](variables.tf#L222) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L237) | Project ID. | string | ✓ | | +| [region](variables.tf#L242) | Dataproc region. | string | ✓ | | | [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | | [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L199) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [labels](variables.tf#L206) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | -| [prefix](variables.tf#L217) | Optional prefix used to generate project id and name. | string | | null | -| [service_account](variables.tf#L237) | Service account to set on the Dataproc cluster. | string | | null | +| [iam_members](variables.tf#L206) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [labels](variables.tf#L216) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [prefix](variables.tf#L227) | Optional prefix used to generate project id and name. | string | | null | +| [service_account](variables.tf#L247) | Service account to set on the Dataproc cluster. | string | | null | ## Outputs @@ -165,5 +186,4 @@ module "processing-dp-cluster" { | [id](outputs.tf#L29) | Fully qualified cluster id. | | | [instance_names](outputs.tf#L34) | List of instance names which have been assigned to the cluster. | | | [name](outputs.tf#L43) | The name of the cluster. | | - diff --git a/modules/dataproc/iam.tf b/modules/dataproc/iam.tf index e44c2a63..84d49f35 100644 --- a/modules/dataproc/iam.tf +++ b/modules/dataproc/iam.tf @@ -65,3 +65,11 @@ resource "google_dataproc_cluster_iam_member" "additive" { role = each.value.role member = each.value.member } + +resource "google_dataproc_cluster_iam_member" "members" { + for_each = var.iam_members + project = var.project_id + cluster = google_dataproc_cluster.cluster.name + role = each.value.role + member = each.value.member +} diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index 926169b9..b28b9c27 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -203,6 +203,16 @@ variable "iam_additive" { nullable = false } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "labels" { description = "The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs." type = map(string) diff --git a/modules/folder/README.md b/modules/folder/README.md index 85f00c3e..b46c5e8b 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -41,15 +41,21 @@ module "folder" { "user:am1@example.org" = ["roles/storage.admin"] "user:am2@example.org" = ["roles/storage.objectViewer"] } + iam_members = { + am1-storage-admin = { + member = "user:am1@example.org" + role = "roles/storage.admin" + } + } } -# tftest modules=1 resources=9 inventory=iam.yaml +# tftest modules=1 resources=10 inventory=iam.yaml ``` ## IAM -There are three mutually exclusive ways at the role level of managing IAM in this module +There are four three exclusive ways at the role level of managing IAM in this module -- non-authoritative via the `iam_additive` and `iam_additive_members` variables, where bindings created outside this module will coexist with those managed here +- non-authoritative via the `iam_additive`, `iam_additive_members` and `iam_members` variables, where bindings created outside this module will coexist with those managed here - authoritative via the `group_iam` and `iam` variables, where bindings created outside this module (eg in the console) will be removed at each `terraform apply` cycle if the same role is also managed here - authoritative policy via the `iam_policy` variable, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role @@ -319,16 +325,17 @@ module "folder" { | [iam](variables.tf#L44) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L51) | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L58) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_policy](variables.tf#L65) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [id](variables.tf#L71) | Folder ID in case you use folder_create=false. | string | | null | -| [logging_data_access](variables.tf#L77) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L92) | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L99) | Logging sinks to create for the organization. | map(object({…})) | | {} | -| [name](variables.tf#L129) | Folder name. | string | | null | -| [org_policies](variables.tf#L135) | Organization policies applied to this folder keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L162) | Path containing org policies in YAML format. | string | | null | -| [parent](variables.tf#L168) | Parent in folders/folder_id or organizations/org_id format. | string | | null | -| [tag_bindings](variables.tf#L178) | Tag bindings for this folder, in key => tag value id format. | map(string) | | null | +| [iam_members](variables.tf#L65) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_policy](variables.tf#L75) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | +| [id](variables.tf#L81) | Folder ID in case you use folder_create=false. | string | | null | +| [logging_data_access](variables.tf#L87) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L102) | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L109) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [name](variables.tf#L139) | Folder name. | string | | null | +| [org_policies](variables.tf#L145) | Organization policies applied to this folder keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L172) | Path containing org policies in YAML format. | string | | null | +| [parent](variables.tf#L178) | Parent in folders/folder_id or organizations/org_id format. | string | | null | +| [tag_bindings](variables.tf#L188) | Tag bindings for this folder, in key => tag value id format. | map(string) | | null | ## Outputs diff --git a/modules/folder/iam.tf b/modules/folder/iam.tf index 5afae6f4..6b8fc1b1 100644 --- a/modules/folder/iam.tf +++ b/modules/folder/iam.tf @@ -64,6 +64,13 @@ resource "google_folder_iam_member" "additive" { member = each.value.member } +resource "google_folder_iam_member" "members" { + for_each = var.iam_members + folder = local.folder.name + role = each.value.role + member = each.value.member +} + resource "google_folder_iam_policy" "authoritative" { count = var.iam_policy != null ? 1 : 0 folder = local.folder.name diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf index 6fccce15..be7aad32 100644 --- a/modules/folder/variables.tf +++ b/modules/folder/variables.tf @@ -62,6 +62,16 @@ variable "iam_additive_members" { nullable = false } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "iam_policy" { description = "IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution." type = map(list(string)) diff --git a/modules/iam-service-account/README.md b/modules/iam-service-account/README.md index c1303386..061b651e 100644 --- a/modules/iam-service-account/README.md +++ b/modules/iam-service-account/README.md @@ -27,7 +27,6 @@ module "myproject-default-service-accounts" { ``` - ## Files | name | description | resources | @@ -42,8 +41,8 @@ module "myproject-default-service-accounts" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L91) | Name of the service account to create. | string | ✓ | | -| [project_id](variables.tf#L106) | Project id where service account will be created. | string | ✓ | | +| [name](variables.tf#L101) | Name of the service account to create. | string | ✓ | | +| [project_id](variables.tf#L116) | Project id where service account will be created. | string | ✓ | | | [description](variables.tf#L17) | Optional description. | string | | null | | [display_name](variables.tf#L23) | Display name of the service account to create. | string | | "Terraform-managed." | | [generate_key](variables.tf#L29) | Generate a key for service account. | bool | | false | @@ -51,13 +50,14 @@ module "myproject-default-service-accounts" { | [iam_additive](variables.tf#L42) | IAM additive bindings on the service account in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_billing_roles](variables.tf#L49) | Billing account roles granted to this service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | | [iam_folder_roles](variables.tf#L56) | Folder roles granted to this service account, by folder id. Non-authoritative. | map(list(string)) | | {} | -| [iam_organization_roles](variables.tf#L63) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | -| [iam_project_roles](variables.tf#L70) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | -| [iam_sa_roles](variables.tf#L77) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | -| [iam_storage_roles](variables.tf#L84) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | -| [prefix](variables.tf#L96) | Prefix applied to service account names. | string | | null | -| [public_keys_directory](variables.tf#L111) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | -| [service_account_create](variables.tf#L117) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | +| [iam_members](variables.tf#L63) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_organization_roles](variables.tf#L73) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | +| [iam_project_roles](variables.tf#L80) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | +| [iam_sa_roles](variables.tf#L87) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | +| [iam_storage_roles](variables.tf#L94) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | +| [prefix](variables.tf#L106) | Prefix applied to service account names. | string | | null | +| [public_keys_directory](variables.tf#L121) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | +| [service_account_create](variables.tf#L127) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | ## Outputs @@ -70,5 +70,4 @@ module "myproject-default-service-accounts" { | [name](outputs.tf#L48) | Service account name. | | | [service_account](outputs.tf#L57) | Service account resource. | | | [service_account_credentials](outputs.tf#L62) | Service account json credential templates for uploaded public keys data. | | - diff --git a/modules/iam-service-account/iam.tf b/modules/iam-service-account/iam.tf index 02c879d9..ae388c8e 100644 --- a/modules/iam-service-account/iam.tf +++ b/modules/iam-service-account/iam.tf @@ -134,6 +134,13 @@ resource "google_service_account_iam_member" "additive" { member = local.resource_iam_email } +resource "google_service_account_iam_member" "members" { + for_each = var.iam_members + service_account_id = each.value.entity + role = each.value.role + member = each.value.member +} + resource "google_storage_bucket_iam_member" "bucket-roles" { for_each = { for pair in local.iam_storage_pairs : diff --git a/modules/iam-service-account/variables.tf b/modules/iam-service-account/variables.tf index a9f60bf2..3594fbbc 100644 --- a/modules/iam-service-account/variables.tf +++ b/modules/iam-service-account/variables.tf @@ -60,6 +60,16 @@ variable "iam_folder_roles" { nullable = false } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "iam_organization_roles" { description = "Organization roles granted to this service account, by organization id. Non-authoritative." type = map(list(string)) diff --git a/modules/kms/README.md b/modules/kms/README.md index 446325d8..7b8eb487 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -4,6 +4,16 @@ This module allows creating and managing KMS crypto keys and IAM bindings at bot When using an existing keyring be mindful about applying IAM bindings, as all bindings used by this module are authoritative, and you might inadvertently override bindings managed by the keyring creator. + +- [Protecting against destroy](#protecting-against-destroy) +- [Examples](#examples) + - [Using an existing keyring](#using-an-existing-keyring) + - [Keyring creation and crypto key rotation and IAM roles](#keyring-creation-and-crypto-key-rotation-and-iam-roles) + - [Crypto key purpose](#crypto-key-purpose) +- [Variables](#variables) +- [Outputs](#outputs) + + ## Protecting against destroy In this module **no lifecycle blocks are set on resources to prevent destroy**, in order to allow for experimentation and testing where rapid `apply`/`destroy` cycles are needed. If you plan on using this module to manage non-development resources, **clone it and uncomment the lifecycle blocks** found in `main.tf`. @@ -49,6 +59,13 @@ module "kms" { ] } } + key_iam_members = { + key-b-am1 = { + key = "key-b" + member = "user:am1@example.com" + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + } + } keyring = { location = "europe-west1", name = "test" } keys = { key-a = null @@ -56,7 +73,7 @@ module "kms" { key-c = { rotation_period = null, labels = { env = "test" } } } } -# tftest modules=1 resources=9 inventory=basic.yaml +# tftest modules=1 resources=10 ``` ### Crypto key purpose @@ -80,22 +97,23 @@ module "kms" { # tftest modules=1 resources=4 ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [keyring](variables.tf#L70) | Keyring attributes. | object({…}) | ✓ | | -| [project_id](variables.tf#L93) | Project id where the keyring will be created. | string | ✓ | | +| [keyring](variables.tf#L91) | Keyring attributes. | object({…}) | ✓ | | +| [project_id](variables.tf#L114) | Project id where the keyring will be created. | string | ✓ | | | [iam](variables.tf#L17) | Keyring IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L23) | Keyring IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [key_iam](variables.tf#L29) | Key IAM bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [key_iam_additive](variables.tf#L35) | Key IAM additive bindings in {ROLE => [MEMBERS]} format. | map(map(list(string))) | | {} | -| [key_purpose](variables.tf#L41) | Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | map(object({…})) | | {} | -| [key_purpose_defaults](variables.tf#L53) | Defaults used for key purpose when not defined at the key level. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | object({…}) | | {…} | -| [keyring_create](variables.tf#L78) | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | -| [keys](variables.tf#L84) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | -| [tag_bindings](variables.tf#L98) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | null | +| [iam_members](variables.tf#L29) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [key_iam](variables.tf#L39) | Key IAM bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [key_iam_additive](variables.tf#L45) | Key IAM additive bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [key_iam_members](variables.tf#L51) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [key_purpose](variables.tf#L62) | Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | map(object({…})) | | {} | +| [key_purpose_defaults](variables.tf#L74) | Defaults used for key purpose when not defined at the key level. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | object({…}) | | {…} | +| [keyring_create](variables.tf#L99) | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | +| [keys](variables.tf#L105) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L119) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | null | ## Outputs @@ -107,5 +125,4 @@ module "kms" { | [keys](outputs.tf#L44) | Key resources. | | | [location](outputs.tf#L52) | Keyring location. | | | [name](outputs.tf#L60) | Keyring name. | | - diff --git a/modules/kms/iam.tf b/modules/kms/iam.tf new file mode 100644 index 00000000..3da20d08 --- /dev/null +++ b/modules/kms/iam.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + iam_additive_members = flatten([ + for role, members in var.iam_additive : [ + for member in members : { + member = member + role = role + } + ] + ]) + key_iam_additive_members = flatten([ + for key, roles in var.key_iam_additive : [ + for role, members in roles : [ + for member in members : { + key = key + member = member + role = role + } + ] + ] + ]) + key_iam_members = flatten([ + for key, roles in var.key_iam : [ + for role, members in roles : { + key = key + role = role + members = members + } + ] + ]) +} + +resource "google_kms_key_ring_iam_binding" "default" { + for_each = var.iam + key_ring_id = local.keyring.id + role = each.key + members = each.value +} + +resource "google_kms_key_ring_iam_member" "default" { + for_each = { + for binding in local.iam_additive_members : + "${binding.role}${binding.member}" => binding + } + key_ring_id = local.keyring.id + role = each.value.role + member = each.value.member +} + +resource "google_kms_key_ring_iam_member" "members" { + for_each = var.iam_members + key_ring_id = local.keyring.id + role = each.value.role + member = each.value.member +} + +resource "google_kms_crypto_key_iam_binding" "default" { + for_each = { + for binding in local.key_iam_members : + "${binding.key}.${binding.role}" => binding + } + role = each.value.role + crypto_key_id = google_kms_crypto_key.default[each.value.key].id + members = each.value.members +} + +resource "google_kms_crypto_key_iam_member" "default" { + for_each = { + for binding in local.key_iam_additive_members : + "${binding.key}.${binding.role}${binding.member}" => binding + } + role = each.value.role + crypto_key_id = google_kms_crypto_key.default[each.value.key].id + member = each.value.member +} + +resource "google_kms_crypto_key_iam_member" "members" { + for_each = var.key_iam_members + crypto_key_id = google_kms_crypto_key.default[each.value.key].id + role = each.value.role + member = each.value.member +} diff --git a/modules/kms/main.tf b/modules/kms/main.tf index 88ac158e..26624f15 100644 --- a/modules/kms/main.tf +++ b/modules/kms/main.tf @@ -15,34 +15,6 @@ */ locals { - iam_additive_members = flatten([ - for role, members in var.iam_additive : [ - for member in members : { - member = member - role = role - } - ] - ]) - key_iam_additive_members = flatten([ - for key, roles in var.key_iam_additive : [ - for role, members in roles : [ - for member in members : { - key = key - member = member - role = role - } - ] - ] - ]) - key_iam_members = flatten([ - for key, roles in var.key_iam : [ - for role, members in roles : { - key = key - role = role - members = members - } - ] - ]) key_purpose = { for key, attrs in var.keys : key => try( var.key_purpose[key], var.key_purpose_defaults @@ -69,23 +41,6 @@ resource "google_kms_key_ring" "default" { location = var.keyring.location } -resource "google_kms_key_ring_iam_binding" "default" { - for_each = var.iam - key_ring_id = local.keyring.id - role = each.key - members = each.value -} - -resource "google_kms_key_ring_iam_member" "default" { - for_each = { - for binding in local.iam_additive_members : - "${binding.role}${binding.member}" => binding - } - key_ring_id = local.keyring.id - role = each.value.role - member = each.value.member -} - resource "google_kms_crypto_key" "default" { for_each = var.keys key_ring = local.keyring.id @@ -101,23 +56,3 @@ resource "google_kms_crypto_key" "default" { } } } - -resource "google_kms_crypto_key_iam_binding" "default" { - for_each = { - for binding in local.key_iam_members : - "${binding.key}.${binding.role}" => binding - } - role = each.value.role - crypto_key_id = google_kms_crypto_key.default[each.value.key].id - members = each.value.members -} - -resource "google_kms_crypto_key_iam_member" "default" { - for_each = { - for binding in local.key_iam_additive_members : - "${binding.key}.${binding.role}${binding.member}" => binding - } - role = each.value.role - crypto_key_id = google_kms_crypto_key.default[each.value.key].id - member = each.value.member -} diff --git a/modules/kms/variables.tf b/modules/kms/variables.tf index f5f0fb1b..01e96c04 100644 --- a/modules/kms/variables.tf +++ b/modules/kms/variables.tf @@ -26,6 +26,16 @@ variable "iam_additive" { default = {} } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "key_iam" { description = "Key IAM bindings in {KEY => {ROLE => [MEMBERS]}} format." type = map(map(list(string))) @@ -33,11 +43,22 @@ variable "key_iam" { } variable "key_iam_additive" { - description = "Key IAM additive bindings in {ROLE => [MEMBERS]} format." + description = "Key IAM additive bindings in {KEY => {ROLE => [MEMBERS]}} format." type = map(map(list(string))) default = {} } +variable "key_iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + key = string + member = string + role = string + })) + nullable = false + default = {} +} + variable "key_purpose" { description = "Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required." type = map(object({ diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 4b8c4f8d..8106e1d9 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -131,8 +131,15 @@ module "vpc" { ] } } + subnet_iam_members = { + subnet-2-am1 = { + member = "user:am1@example.com" + role = "roles/compute.networkUser" + subnet = "europe-west1/subnet-2" + } + } } -# tftest modules=1 resources=8 inventory=subnet-iam.yaml +# tftest modules=1 resources=9 inventory=subnet-iam.yaml ``` ### Peering @@ -534,10 +541,11 @@ module "vpc" { | [shared_vpc_service_projects](variables.tf#L161) | Shared VPC service projects to register with this host. | list(string) | | [] | | [subnet_iam](variables.tf#L167) | Subnet IAM bindings in {REGION/NAME => {ROLE => [MEMBERS]} format. | map(map(list(string))) | | {} | | [subnet_iam_additive](variables.tf#L173) | Subnet IAM additive bindings in {REGION/NAME => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [subnets](variables.tf#L180) | Subnet configuration. | list(object({…})) | | [] | -| [subnets_proxy_only](variables.tf#L206) | List of proxy-only subnets for Regional HTTPS or Internal HTTPS load balancers. Note: Only one proxy-only subnet for each VPC network in each region can be active. | list(object({…})) | | [] | -| [subnets_psc](variables.tf#L218) | List of subnets for Private Service Connect service producers. | list(object({…})) | | [] | -| [vpc_create](variables.tf#L229) | Create VPC. When set to false, uses a data source to reference existing VPC. | bool | | true | +| [subnet_iam_members](variables.tf#L180) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [subnets](variables.tf#L191) | Subnet configuration. | list(object({…})) | | [] | +| [subnets_proxy_only](variables.tf#L217) | List of proxy-only subnets for Regional HTTPS or Internal HTTPS load balancers. Note: Only one proxy-only subnet for each VPC network in each region can be active. | list(object({…})) | | [] | +| [subnets_psc](variables.tf#L229) | List of subnets for Private Service Connect service producers. | list(object({…})) | | [] | +| [vpc_create](variables.tf#L240) | Create VPC. When set to false, uses a data source to reference existing VPC. | bool | | true | ## Outputs diff --git a/modules/net-vpc/subnets.tf b/modules/net-vpc/subnets.tf index 262c22da..16f6398c 100644 --- a/modules/net-vpc/subnets.tf +++ b/modules/net-vpc/subnets.tf @@ -189,3 +189,12 @@ resource "google_compute_subnetwork_iam_member" "binding" { role = each.value.role member = each.value.member } + +resource "google_compute_subnetwork_iam_member" "members" { + for_each = var.subnet_iam_members + project = var.project_id + subnetwork = google_compute_subnetwork.subnetwork[each.value.subnet].name + region = google_compute_subnetwork.subnetwork[each.value.subnet].region + role = each.value.role + member = each.value.member +} diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index 733692d8..4e114dfe 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -177,6 +177,17 @@ variable "subnet_iam_additive" { nullable = false } +variable "subnet_iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + subnet = string + })) + nullable = false + default = {} +} + variable "subnets" { description = "Subnet configuration." type = list(object({ diff --git a/modules/organization/README.md b/modules/organization/README.md index 7c4271ab..d232dfd4 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -45,6 +45,12 @@ module "org" { iam_additive_members = { "user:compute@example.org" = ["roles/compute.admin", "roles/container.viewer"] } + iam_members = { + am1-storage-admin = { + member = "user:am1@example.org" + role = "roles/storage.admin" + } + } tags = { allowexternal = { description = "Allow external identities." @@ -115,14 +121,14 @@ module "org" { } } } -# tftest modules=1 resources=16 inventory=basic.yaml +# tftest modules=1 resources=17 inventory=basic.yaml ``` ## IAM -There are three mutually exclusive ways of managing IAM in this module +There are three mutually exclusive ways at the role level of managing IAM in this module -- non-authoritative via the `iam_additive` and `iam_additive_members` variables, where bindings created outside this module will coexist with those managed here +- non-authoritative via the `iam_additive`, `iam_additive_members` and `iam_members` variables, where bindings created outside this module will coexist with those managed here - authoritative via the `group_iam` and `iam` variables, where bindings created outside this module (eg in the console) will be removed at each `terraform apply` cycle if the same role is also managed here - authoritative policy via the `iam_policy` variable, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role @@ -469,7 +475,7 @@ module "org" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [organization_id](variables.tf#L199) | Organization id in organizations/nnnnnn format. | string | ✓ | | +| [organization_id](variables.tf#L209) | Organization id in organizations/nnnnnn format. | string | ✓ | | | [contacts](variables.tf#L17) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | | [custom_roles](variables.tf#L24) | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | | [firewall_policy_associations](variables.tf#L31) | Hierarchical firewall policies to associate to this folder, in association name => policy id format. | map(string) | | {} | @@ -477,17 +483,18 @@ module "org" { | [iam](variables.tf#L45) | IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L52) | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L59) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_policy](variables.tf#L66) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [logging_data_access](variables.tf#L72) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L87) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L94) | Logging sinks to create for the organization. | map(object({…})) | | {} | -| [network_tags](variables.tf#L124) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | -| [org_policies](variables.tf#L146) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L173) | Path containing org policies in YAML format. | string | | null | -| [org_policy_custom_constraints](variables.tf#L179) | Organization policy custom constraints keyed by constraint name. | map(object({…})) | | {} | -| [org_policy_custom_constraints_data_path](variables.tf#L193) | Path containing org policy custom constraints in YAML format. | string | | null | -| [tag_bindings](variables.tf#L208) | Tag bindings for this organization, in key => tag value id format. | map(string) | | null | -| [tags](variables.tf#L214) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | +| [iam_members](variables.tf#L66) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_policy](variables.tf#L76) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | +| [logging_data_access](variables.tf#L82) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L97) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L104) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [network_tags](variables.tf#L134) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | +| [org_policies](variables.tf#L156) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L183) | Path containing org policies in YAML format. | string | | null | +| [org_policy_custom_constraints](variables.tf#L189) | Organization policy custom constraints keyed by constraint name. | map(object({…})) | | {} | +| [org_policy_custom_constraints_data_path](variables.tf#L203) | Path containing org policy custom constraints in YAML format. | string | | null | +| [tag_bindings](variables.tf#L218) | Tag bindings for this organization, in key => tag value id format. | map(string) | | null | +| [tags](variables.tf#L224) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | ## Outputs diff --git a/modules/organization/iam.tf b/modules/organization/iam.tf index 410c9ece..f5d3b2d7 100644 --- a/modules/organization/iam.tf +++ b/modules/organization/iam.tf @@ -73,6 +73,13 @@ resource "google_organization_iam_member" "additive" { member = each.value.member } +resource "google_organization_iam_member" "members" { + for_each = var.iam_members + org_id = local.organization_id_numeric + role = each.value.role + member = each.value.member +} + resource "google_organization_iam_policy" "authoritative" { count = var.iam_policy != null ? 1 : 0 org_id = local.organization_id_numeric diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf index b9ba5820..65f0672b 100644 --- a/modules/organization/variables.tf +++ b/modules/organization/variables.tf @@ -63,6 +63,16 @@ variable "iam_additive_members" { nullable = false } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "iam_policy" { description = "IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution." type = map(list(string)) diff --git a/modules/project/README.md b/modules/project/README.md index a39df3ce..8a867b51 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -10,7 +10,9 @@ This module implements the creation and management of one GCP project including - [IAM](#iam) - [Authoritative IAM](#authoritative-iam) - [Additive IAM](#additive-iam) - - [Additive IAM by Member](#additive-iam-by-member) + - [Additive IAM by Role](#additive-iam-by-role) + - [Additive IAM by Principal](#additive-iam-by-principal) + - [Additive IAM by Binding](#additive-iam-by-binding) - [Service Identities and Authoritative IAM](#service-identities-and-authoritative-iam) - [Using Shortcodes for Service Identities in Additive Iam](#using-shortcodes-for-service-identities-in-additive-iam) - [Service Identities Requiring Manual Iam Grants](#service-identities-requiring-manual-iam-grants) @@ -49,7 +51,7 @@ module "project" { IAM is managed via several variables that implement different levels of control: - `group_iam` and `iam` configure authoritative bindings that manage individual roles exclusively, mapping to the [`google_project_iam_binding`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_binding) resource -- `iam_additive` and `iam_additive_members` configure additive bindings that only manage individual role/member pairs, mapping to the [`google_project_iam_member`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_member) resource +- `iam_additive`, `iam_additive_members` and `iam_members` configure additive bindings that only manage individual role/member pairs, mapping to the [`google_project_iam_member`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_member) resource - `iam_policy` which controls the entire IAM policy for the project, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role The authoritative and additive approaches can be used together, provided different roles are managed by each. The IAM policy is incompatible with the other approaches, and must be used with extreme care. @@ -109,6 +111,10 @@ module "project" { Additive IAM is typically used where bindings for specific roles are controlled by different modules or in different Terraform stages. One example is when the project is created by one team but a different team manages service account creation for the project, and some of the project-level roles overlap in the two configurations. +#### Additive IAM by Role + +Additive IAM is supported via the `iam_additive` variable which is keyed by role: + ```hcl module "project" { source = "./fabric/modules/project" @@ -129,7 +135,9 @@ module "project" { # tftest modules=1 resources=5 inventory=iam-additive.yaml ``` -### Additive IAM by Member +#### Additive IAM by Principal + +Additive IAM is also supported via the `iam_additive_members` variable which is keyed by principal: ```hcl module "project" { @@ -144,6 +152,33 @@ module "project" { # tftest modules=1 resources=4 inventory=iam-additive-members.yaml ``` +#### Additive IAM by Binding + +When the above approaches to additive IAM are unworkable due to dynamically generated principals, the `iam_members` variable allows specifying individual role/principal pairs using arbitrary keys: + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "project-example" + iam_members = { + one-owner = { + member = "user:one@example.org" + role = "roles/owner" + } + two-viewer = { + member = "user:two@example.org" + role = "roles/viewer" + } + two-compute-admin = { + member = "user:two@example.org" + role = "roles/compute.admin" + } + } + +} +# tftest modules=1 resources=4 inventory=iam-members.yaml +``` + ### Service Identities and Authoritative IAM As mentioned above, there are cases where authoritative management of specific IAM roles results in removal of default bindings from service identities. One example is outlined below, with a simple workaround leveraging the `service_accounts` output to identify the service identity. A full list of service identities and their roles can be found [here](https://cloud.google.com/iam/docs/service-agents). @@ -260,6 +295,7 @@ module "service-project" { ``` The module allows also granting necessary permissions in host project to service identities by specifying which services will be used in service project in `grant_iam_for_services`. + ```hcl module "host-project" { source = "./fabric/modules/project" @@ -622,7 +658,7 @@ output "compute_robot" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L161) | Project name and id suffix. | string | ✓ | | +| [name](variables.tf#L171) | Project name and id suffix. | string | ✓ | | | [auto_create_network](variables.tf#L17) | Whether to create the default network for the project. | bool | | false | | [billing_account](variables.tf#L23) | Billing account id. | string | | null | | [contacts](variables.tf#L29) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | @@ -633,30 +669,31 @@ output "compute_robot" { | [iam](variables.tf#L62) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L69) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L76) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_policy](variables.tf#L82) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [labels](variables.tf#L88) | Resource labels. | map(string) | | {} | -| [lien_reason](variables.tf#L95) | If non-empty, creates a project lien with this description. | string | | "" | -| [logging_data_access](variables.tf#L101) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L116) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L123) | Logging sinks to create for this project. | map(object({…})) | | {} | -| [metric_scopes](variables.tf#L154) | List of projects that will act as metric scopes for this project. | list(string) | | [] | -| [org_policies](variables.tf#L166) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L193) | Path containing org policies in YAML format. | string | | null | -| [oslogin](variables.tf#L199) | Enable OS Login. | bool | | false | -| [oslogin_admins](variables.tf#L205) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | -| [oslogin_users](variables.tf#L213) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | -| [parent](variables.tf#L220) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | -| [prefix](variables.tf#L230) | Optional prefix used to generate project id and name. | string | | null | -| [project_create](variables.tf#L240) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | -| [service_config](variables.tf#L246) | Configure service API activation. | object({…}) | | {…} | -| [service_encryption_key_ids](variables.tf#L258) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | -| [service_perimeter_bridges](variables.tf#L265) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | -| [service_perimeter_standard](variables.tf#L272) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | -| [services](variables.tf#L278) | Service APIs to enable. | list(string) | | [] | -| [shared_vpc_host_config](variables.tf#L284) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | -| [shared_vpc_service_config](variables.tf#L293) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | -| [skip_delete](variables.tf#L315) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | -| [tag_bindings](variables.tf#L321) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | +| [iam_members](variables.tf#L82) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_policy](variables.tf#L92) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | +| [labels](variables.tf#L98) | Resource labels. | map(string) | | {} | +| [lien_reason](variables.tf#L105) | If non-empty, creates a project lien with this description. | string | | "" | +| [logging_data_access](variables.tf#L111) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L126) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L133) | Logging sinks to create for this project. | map(object({…})) | | {} | +| [metric_scopes](variables.tf#L164) | List of projects that will act as metric scopes for this project. | list(string) | | [] | +| [org_policies](variables.tf#L176) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L203) | Path containing org policies in YAML format. | string | | null | +| [oslogin](variables.tf#L209) | Enable OS Login. | bool | | false | +| [oslogin_admins](variables.tf#L215) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | +| [oslogin_users](variables.tf#L223) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | +| [parent](variables.tf#L230) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | +| [prefix](variables.tf#L240) | Optional prefix used to generate project id and name. | string | | null | +| [project_create](variables.tf#L250) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | +| [service_config](variables.tf#L256) | Configure service API activation. | object({…}) | | {…} | +| [service_encryption_key_ids](variables.tf#L268) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | +| [service_perimeter_bridges](variables.tf#L275) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | +| [service_perimeter_standard](variables.tf#L282) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | +| [services](variables.tf#L288) | Service APIs to enable. | list(string) | | [] | +| [shared_vpc_host_config](variables.tf#L294) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | +| [shared_vpc_service_config](variables.tf#L303) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | +| [skip_delete](variables.tf#L325) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | +| [tag_bindings](variables.tf#L331) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | ## Outputs diff --git a/modules/project/iam.tf b/modules/project/iam.tf index 2ee8a63c..7b67d895 100644 --- a/modules/project/iam.tf +++ b/modules/project/iam.tf @@ -97,6 +97,17 @@ resource "google_project_iam_member" "additive" { ] } +resource "google_project_iam_member" "members" { + for_each = var.iam_members + project = local.project.project_id + role = each.value.role + member = each.value.member + depends_on = [ + google_project_service.project_services, + google_project_iam_custom_role.roles + ] +} + resource "google_project_iam_member" "oslogin_iam_serviceaccountuser" { for_each = var.oslogin ? toset(distinct(concat(var.oslogin_admins, var.oslogin_users))) : toset([]) project = local.project.project_id diff --git a/modules/project/variables.tf b/modules/project/variables.tf index eb6b5da7..4ab51062 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -79,6 +79,16 @@ variable "iam_additive_members" { default = {} } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "iam_policy" { description = "IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution." type = map(list(string)) diff --git a/modules/source-repository/README.md b/modules/source-repository/README.md index 17662fea..4cfae8ea 100644 --- a/modules/source-repository/README.md +++ b/modules/source-repository/README.md @@ -2,6 +2,15 @@ This module allows managing a single Cloud Source Repository, including IAM bindings and basic Cloud Build triggers. + +- [Examples](#examples) + - [Repository with IAM](#repository-with-iam) + - [Repository with Cloud Build trigger](#repository-with-cloud-build-trigger) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + ## Examples ### Repository with IAM @@ -14,8 +23,14 @@ module "repo" { iam = { "roles/source.reader" = ["user:foo@example.com"] } + iam_members = { + am1-reader = { + member = "user:am1@example.com" + role = "roles/source.reader" + } + } } -# tftest modules=1 resources=2 inventory=simple.yaml +# tftest modules=1 resources=3 inventory=simple.yaml ``` ### Repository with Cloud Build trigger @@ -46,7 +61,6 @@ module "repo" { - ## Files | name | description | resources | @@ -61,13 +75,14 @@ module "repo" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L44) | Repository name. | string | ✓ | | -| [project_id](variables.tf#L49) | Project used for resources. | string | ✓ | | +| [name](variables.tf#L54) | Repository name. | string | ✓ | | +| [project_id](variables.tf#L59) | Project used for resources. | string | ✓ | | | [group_iam](variables.tf#L17) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L24) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L31) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L38) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [triggers](variables.tf#L54) | Cloud Build triggers. | map(object({…})) | | {} | +| [iam_members](variables.tf#L44) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [triggers](variables.tf#L64) | Cloud Build triggers. | map(object({…})) | | {} | ## Outputs @@ -76,5 +91,4 @@ module "repo" { | [id](outputs.tf#L17) | Fully qualified repository id. | | | [name](outputs.tf#L22) | Repository name. | | | [url](outputs.tf#L27) | Repository URL. | | - diff --git a/modules/source-repository/iam.tf b/modules/source-repository/iam.tf index e5c3ec49..f1ec3ebc 100644 --- a/modules/source-repository/iam.tf +++ b/modules/source-repository/iam.tf @@ -65,3 +65,11 @@ resource "google_sourcerepo_repository_iam_member" "additive" { role = each.value.role member = each.value.member } + +resource "google_sourcerepo_repository_iam_member" "members" { + for_each = var.iam_members + project = var.project_id + repository = google_sourcerepo_repository.default.name + role = each.value.role + member = each.value.member +} diff --git a/modules/source-repository/variables.tf b/modules/source-repository/variables.tf index 587b0f6d..74e44aa4 100644 --- a/modules/source-repository/variables.tf +++ b/modules/source-repository/variables.tf @@ -41,6 +41,16 @@ variable "iam_additive_members" { default = {} } +variable "iam_members" { + description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + variable "name" { description = "Repository name." type = string diff --git a/tests/modules/dataplex_datascan/examples/datascan_iam.yaml b/tests/modules/dataplex_datascan/examples/datascan_iam.yaml index b2d7a985..28885cfe 100644 --- a/tests/modules/dataplex_datascan/examples/datascan_iam.yaml +++ b/tests/modules/dataplex_datascan/examples/datascan_iam.yaml @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + values: module.dataplex-datascan.google_dataplex_datascan.datascan: data: @@ -57,11 +58,19 @@ values: - group:user-group@example.com project: my-project-name role: roles/dataplex.dataScanViewer + module.dataplex-datascan.google_dataplex_datascan_iam_member.members["am1-viewer"]: + condition: [] + data_scan_id: test-datascan + location: us-central1 + member: user:am1@example.com + project: my-project-name + role: roles/dataplex.dataScanViewer counts: google_dataplex_datascan: 1 google_dataplex_datascan_iam_binding: 3 + google_dataplex_datascan_iam_member: 1 modules: 1 - resources: 4 + resources: 5 -outputs: {} \ No newline at end of file +outputs: {} diff --git a/tests/modules/folder/examples/iam.yaml b/tests/modules/folder/examples/iam.yaml index 6f0fe2e5..541fbf86 100644 --- a/tests/modules/folder/examples/iam.yaml +++ b/tests/modules/folder/examples/iam.yaml @@ -16,6 +16,7 @@ values: module.folder.google_folder.folder[0]: display_name: Folder name parent: organizations/1234567890 + timeouts: null module.folder.google_folder_iam_binding.authoritative["roles/owner"]: condition: [] members: @@ -52,8 +53,16 @@ values: condition: [] member: user:am2@example.org role: roles/storage.objectViewer + module.folder.google_folder_iam_member.members["am1-storage-admin"]: + condition: [] + member: user:am1@example.org + role: roles/storage.admin counts: google_folder: 1 google_folder_iam_binding: 3 - google_folder_iam_member: 5 + google_folder_iam_member: 6 + modules: 1 + resources: 10 + +outputs: {} diff --git a/tests/modules/kms/examples/basic.yaml b/tests/modules/kms/examples/basic.yaml index acd7ac29..e29297a1 100644 --- a/tests/modules/kms/examples/basic.yaml +++ b/tests/modules/kms/examples/basic.yaml @@ -19,12 +19,14 @@ values: purpose: ENCRYPT_DECRYPT rotation_period: null skip_initial_version_creation: null + timeouts: null module.kms.google_kms_crypto_key.default["key-b"]: labels: null name: key-b purpose: ENCRYPT_DECRYPT rotation_period: 604800s skip_initial_version_creation: null + timeouts: null module.kms.google_kms_crypto_key.default["key-c"]: labels: env: test @@ -32,23 +34,29 @@ values: purpose: ENCRYPT_DECRYPT rotation_period: null skip_initial_version_creation: null + timeouts: null module.kms.google_kms_crypto_key_iam_binding.default["key-a.roles/cloudkms.admin"]: condition: [] members: - user:user3@example.com role: roles/cloudkms.admin - module.kms.google_kms_crypto_key_iam_member.default["key-b.roles/cloudkms.cryptoKeyEncrypterDecrypteruser:user4@example.com"]: - condition: [] + ? module.kms.google_kms_crypto_key_iam_member.default["key-b.roles/cloudkms.cryptoKeyEncrypterDecrypteruser:user4@example.com"] + : condition: [] member: user:user4@example.com role: roles/cloudkms.cryptoKeyEncrypterDecrypter - module.kms.google_kms_crypto_key_iam_member.default["key-b.roles/cloudkms.cryptoKeyEncrypterDecrypteruser:user5@example.com"]: - condition: [] + ? module.kms.google_kms_crypto_key_iam_member.default["key-b.roles/cloudkms.cryptoKeyEncrypterDecrypteruser:user5@example.com"] + : condition: [] member: user:user5@example.com role: roles/cloudkms.cryptoKeyEncrypterDecrypter + module.kms.google_kms_crypto_key_iam_member.members["key-b-am1"]: + condition: [] + member: user:am1@example.com + role: roles/cloudkms.cryptoKeyEncrypterDecrypter module.kms.google_kms_key_ring.default[0]: location: europe-west1 name: test project: my-project + timeouts: null module.kms.google_kms_key_ring_iam_member.default["roles/cloudkms.cryptoKeyEncrypterDecrypteruser:user1@example.com"]: condition: [] member: user:user1@example.com @@ -61,6 +69,10 @@ values: counts: google_kms_crypto_key: 3 google_kms_crypto_key_iam_binding: 1 - google_kms_crypto_key_iam_member: 2 + google_kms_crypto_key_iam_member: 3 google_kms_key_ring: 1 google_kms_key_ring_iam_member: 2 + modules: 1 + resources: 10 + +outputs: {} diff --git a/tests/modules/net_vpc/examples/subnet-iam.yaml b/tests/modules/net_vpc/examples/subnet-iam.yaml index 8aa5bed8..2c6dcccc 100644 --- a/tests/modules/net_vpc/examples/subnet-iam.yaml +++ b/tests/modules/net_vpc/examples/subnet-iam.yaml @@ -14,17 +14,63 @@ values: module.vpc.google_compute_network.network[0]: + auto_create_subnetworks: false + delete_default_routes_on_create: false + description: Terraform-managed. + enable_ula_internal_ipv6: null name: my-network + network_firewall_policy_enforcement_order: AFTER_CLASSIC_FIREWALL project: my-project + routing_mode: GLOBAL + timeouts: null + module.vpc.google_compute_route.gateway["private-googleapis"]: + description: Terraform-managed. + dest_range: 199.36.153.8/30 + name: my-network-private-googleapis + next_hop_gateway: default-internet-gateway + next_hop_ilb: null + next_hop_instance: null + next_hop_vpn_tunnel: null + priority: 1000 + project: my-project + tags: null + timeouts: null + module.vpc.google_compute_route.gateway["restricted-googleapis"]: + description: Terraform-managed. + dest_range: 199.36.153.4/30 + name: my-network-restricted-googleapis + next_hop_gateway: default-internet-gateway + next_hop_ilb: null + next_hop_instance: null + next_hop_vpn_tunnel: null + priority: 1000 + project: my-project + tags: null + timeouts: null module.vpc.google_compute_subnetwork.subnetwork["europe-west1/subnet-1"]: + description: Terraform-managed. + ip_cidr_range: 10.0.1.0/24 + ipv6_access_type: null + log_config: [] name: subnet-1 + private_ip_google_access: true project: my-project region: europe-west1 + role: null + secondary_ip_range: [] + timeouts: null module.vpc.google_compute_subnetwork.subnetwork["europe-west1/subnet-2"]: + description: Terraform-managed. + ip_cidr_range: 10.0.1.0/24 + ipv6_access_type: null + log_config: [] name: subnet-2 private_ip_google_access: true project: my-project region: europe-west1 + role: null + secondary_ip_range: [] + timeouts: null module.vpc.google_compute_subnetwork_iam_binding.binding["europe-west1/subnet-1.roles/compute.networkUser"]: condition: [] members: @@ -34,16 +80,23 @@ values: region: europe-west1 role: roles/compute.networkUser subnetwork: subnet-1 - module.vpc.google_compute_subnetwork_iam_member.binding["europe-west1/subnet-2.roles/compute.networkUser.user:user2@example.com"]: - condition: [] + ? module.vpc.google_compute_subnetwork_iam_member.binding["europe-west1/subnet-2.roles/compute.networkUser.group:group2@example.com"] + : condition: [] + member: group:group2@example.com + project: my-project + region: europe-west1 + role: roles/compute.networkUser + subnetwork: subnet-2 + ? module.vpc.google_compute_subnetwork_iam_member.binding["europe-west1/subnet-2.roles/compute.networkUser.user:user2@example.com"] + : condition: [] member: user:user2@example.com project: my-project region: europe-west1 role: roles/compute.networkUser subnetwork: subnet-2 - module.vpc.google_compute_subnetwork_iam_member.binding["europe-west1/subnet-2.roles/compute.networkUser.group:group2@example.com"]: + module.vpc.google_compute_subnetwork_iam_member.members["subnet-2-am1"]: condition: [] - member: group:group2@example.com + member: user:am1@example.com project: my-project region: europe-west1 role: roles/compute.networkUser @@ -51,7 +104,11 @@ values: counts: google_compute_network: 1 + google_compute_route: 2 google_compute_subnetwork: 2 google_compute_subnetwork_iam_binding: 1 - google_compute_subnetwork_iam_member: 2 - google_compute_route: 2 + google_compute_subnetwork_iam_member: 3 + modules: 1 + resources: 9 + +outputs: {} diff --git a/tests/modules/organization/examples/basic.yaml b/tests/modules/organization/examples/basic.yaml index 9960a712..3c8bf8a1 100644 --- a/tests/modules/organization/examples/basic.yaml +++ b/tests/modules/organization/examples/basic.yaml @@ -25,6 +25,7 @@ values: deny_all: null enforce: 'TRUE' values: [] + timeouts: null module.org.google_org_policy_policy.default["compute.skipDefaultNetworkCreation"]: name: organizations/1234567890/policies/compute.skipDefaultNetworkCreation parent: organizations/1234567890 @@ -37,6 +38,7 @@ values: deny_all: null enforce: 'TRUE' values: [] + timeouts: null module.org.google_org_policy_policy.default["compute.trustedImageProjects"]: name: organizations/1234567890/policies/compute.trustedImageProjects parent: organizations/1234567890 @@ -52,6 +54,7 @@ values: - allowed_values: - projects/my-project denied_values: null + timeouts: null module.org.google_org_policy_policy.default["compute.vmExternalIpAccess"]: name: organizations/1234567890/policies/compute.vmExternalIpAccess parent: organizations/1234567890 @@ -64,6 +67,20 @@ values: deny_all: 'TRUE' enforce: null values: [] + timeouts: null + module.org.google_org_policy_policy.default["custom.gkeEnableAutoUpgrade"]: + name: organizations/1234567890/policies/custom.gkeEnableAutoUpgrade + parent: organizations/1234567890 + spec: + - inherit_from_parent: null + reset: null + rules: + - allow_all: null + condition: [] + deny_all: null + enforce: 'TRUE' + values: [] + timeouts: null module.org.google_org_policy_policy.default["iam.allowedPolicyMemberDomains"]: name: organizations/1234567890/policies/iam.allowedPolicyMemberDomains parent: organizations/1234567890 @@ -95,6 +112,7 @@ values: - C0xxxxxxx - C0yyyyyyy denied_values: null + timeouts: null module.org.google_org_policy_policy.default["iam.disableServiceAccountKeyCreation"]: name: organizations/1234567890/policies/iam.disableServiceAccountKeyCreation parent: organizations/1234567890 @@ -107,6 +125,7 @@ values: deny_all: null enforce: 'TRUE' values: [] + timeouts: null module.org.google_org_policy_policy.default["iam.disableServiceAccountKeyUpload"]: name: organizations/1234567890/policies/iam.disableServiceAccountKeyUpload parent: organizations/1234567890 @@ -128,6 +147,7 @@ values: deny_all: null enforce: 'FALSE' values: [] + timeouts: null module.org.google_organization_iam_binding.authoritative["roles/owner"]: condition: [] members: @@ -156,20 +176,34 @@ values: member: user:compute@example.org org_id: '1234567890' role: roles/container.viewer + module.org.google_organization_iam_member.members["am1-storage-admin"]: + condition: [] + member: user:am1@example.org + org_id: '1234567890' + role: roles/storage.admin module.org.google_tags_tag_key.default["allowexternal"]: description: Allow external identities. parent: organizations/1234567890 purpose: null purpose_data: null short_name: allowexternal + timeouts: null module.org.google_tags_tag_value.default["allowexternal/false"]: + description: Managed by the Terraform organization module. short_name: 'false' + timeouts: null module.org.google_tags_tag_value.default["allowexternal/true"]: + description: Managed by the Terraform organization module. short_name: 'true' + timeouts: null counts: google_org_policy_policy: 8 google_organization_iam_binding: 3 - google_organization_iam_member: 2 + google_organization_iam_member: 3 google_tags_tag_key: 1 google_tags_tag_value: 2 + modules: 1 + resources: 17 + +outputs: {} diff --git a/tests/modules/project/examples/iam-members.yaml b/tests/modules/project/examples/iam-members.yaml new file mode 100644 index 00000000..b80fc2bf --- /dev/null +++ b/tests/modules/project/examples/iam-members.yaml @@ -0,0 +1,48 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.project.google_project.project[0]: + auto_create_network: false + billing_account: null + folder_id: null + labels: null + name: project-example + org_id: null + project_id: project-example + skip_delete: false + timeouts: null + module.project.google_project_iam_member.members["one-owner"]: + condition: [] + member: user:one@example.org + project: project-example + role: roles/owner + module.project.google_project_iam_member.members["two-compute-admin"]: + condition: [] + member: user:two@example.org + project: project-example + role: roles/compute.admin + module.project.google_project_iam_member.members["two-viewer"]: + condition: [] + member: user:two@example.org + project: project-example + role: roles/viewer + +counts: + google_project: 1 + google_project_iam_member: 3 + modules: 1 + resources: 4 + +outputs: {} diff --git a/tests/modules/source_repository/examples/simple.yaml b/tests/modules/source_repository/examples/simple.yaml index 41d6aea6..249b6af1 100644 --- a/tests/modules/source_repository/examples/simple.yaml +++ b/tests/modules/source_repository/examples/simple.yaml @@ -17,6 +17,7 @@ values: name: my-repo project: my-project pubsub_configs: [] + timeouts: null module.repo.google_sourcerepo_repository_iam_binding.authoritative["roles/source.reader"]: condition: [] members: @@ -24,7 +25,18 @@ values: project: my-project repository: my-repo role: roles/source.reader + module.repo.google_sourcerepo_repository_iam_member.members["am1-reader"]: + condition: [] + member: user:am1@example.com + project: my-project + repository: my-repo + role: roles/source.reader counts: google_sourcerepo_repository: 1 google_sourcerepo_repository_iam_binding: 1 + google_sourcerepo_repository_iam_member: 1 + modules: 1 + resources: 3 + +outputs: {} From 4c0c9b16f4b63ed482d3f71971ff5cde840eb10d Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 14 Aug 2023 12:07:38 +0200 Subject: [PATCH 06/46] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef1e4e60..f05cd212 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ All notable changes to this project will be documented in this file. ### MODULES +- [[#1589](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1589)] Add new `iam_members` variable to IAM additive module interfaces ([ludoo](https://github.com/ludoo)) +- [[#1588](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1588)] feat: 🎸 (modules/cloudsql-instance): enable require_ssl cert support ([erabusi](https://github.com/erabusi)) - [[#1587](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1587)] **incompatible change:** Fix factory rules key in net firewall policy module ([ludoo](https://github.com/ludoo)) - [[#1578](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1578)] Fix: Instance level stateful disk config ([beardedsamwise](https://github.com/beardedsamwise)) - [[#1582](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1582)] feat(modules/cloud-run): add gen2 exec env support ([LiuVII](https://github.com/LiuVII)) From b6b660f4f3d43a5ca1fdca7fac98ace2be8b06c9 Mon Sep 17 00:00:00 2001 From: erabusi <72961506+erabusi@users.noreply.github.com> Date: Mon, 14 Aug 2023 16:10:25 +0530 Subject: [PATCH 07/46] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20(modules/cloudsql-?= =?UTF-8?q?instance):add=20project=5Fid=20for=20ssl=20cert=20(#1591)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudsql-instance/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/cloudsql-instance/main.tf b/modules/cloudsql-instance/main.tf index 5c40b212..05f150dd 100644 --- a/modules/cloudsql-instance/main.tf +++ b/modules/cloudsql-instance/main.tf @@ -194,6 +194,7 @@ resource "google_sql_user" "users" { resource "google_sql_ssl_cert" "postgres_client_certificates" { for_each = var.postgres_client_certificates != null ? toset(var.postgres_client_certificates) : toset([]) provider = google-beta + project = var.project_id instance = google_sql_database_instance.primary.name common_name = each.key } From f3d5dd8e34a1c3086eb51e34b43ce32f04905692 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Mon, 14 Aug 2023 14:41:33 +0200 Subject: [PATCH 08/46] Added link to GCVE module --- README.md | 2 +- modules/README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bae85ce4..c7b6d94c 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Currently available modules: - **foundational** - [billing budget](./modules/billing-budget), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source) - **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [Internal Application LB](./modules/net-lb-app-int), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC firewall policy](./modules/net-vpc-firewall-policy), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) -- **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool) +- **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) - **data** - [AlloyDB instance](./modules/alloydb-instance), [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub) - **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository) - **security** - [Binauthz](./modules/binauthz/), [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc) diff --git a/modules/README.md b/modules/README.md index 653c25e8..8aab0cb3 100644 --- a/modules/README.md +++ b/modules/README.md @@ -70,6 +70,7 @@ These modules are used in the examples included in this repository. If you are u - [GKE standard cluster](./gke-cluster-standard) - [GKE hub](./gke-hub) - [GKE nodepool](./gke-nodepool) +- [GCVE private cloud](./gcve-private-cloud) ## Data From 2423fd40c110a35cdac6a1105c90b4e30e273c5a Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 15 Aug 2023 12:59:31 +0200 Subject: [PATCH 09/46] Fix FAST CI/CD for Gitlab (#1593) * fix cicd (multitenant untested) * tfdoc * rename allowed_audiences to audiences, align multitenant --- fast/assets/templates/workflow-gitlab.yaml | 58 ++++++++-------- .../0-bootstrap-tenant/README.md | 52 +++++++------- .../0-bootstrap-tenant/cicd.tf | 24 ++++--- .../0-bootstrap-tenant/identity-providers.tf | 19 +++--- .../0-bootstrap-tenant/outputs.tf | 10 +-- .../templates/workflow-gitlab.yaml | 54 +++++++-------- .../0-bootstrap-tenant/variables.tf | 11 +-- .../1-resman-tenant/README.md | 46 ++++++------- .../1-resman-tenant/outputs.tf | 3 + .../templates/workflow-gitlab.yaml | 58 ++++++++-------- .../1-resman-tenant/variables.tf | 2 +- fast/stages/0-bootstrap/README.md | 12 ++-- fast/stages/0-bootstrap/cicd.tf | 15 +++-- fast/stages/0-bootstrap/identity-providers.tf | 2 +- fast/stages/0-bootstrap/outputs.tf | 2 +- .../templates/workflow-gitlab.yaml | 67 +++++++++---------- fast/stages/0-bootstrap/variables.tf | 4 +- fast/stages/1-resman/README.md | 2 +- fast/stages/1-resman/outputs.tf | 4 +- .../1-resman/templates/workflow-gitlab.yaml | 67 +++++++++---------- fast/stages/1-resman/variables.tf | 2 +- 21 files changed, 258 insertions(+), 256 deletions(-) diff --git a/fast/assets/templates/workflow-gitlab.yaml b/fast/assets/templates/workflow-gitlab.yaml index 8981e70b..13057e11 100644 --- a/fast/assets/templates/workflow-gitlab.yaml +++ b/fast/assets/templates/workflow-gitlab.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +13,6 @@ # limitations under the License. default: - before_script: - - echo "$${CI_JOB_JWT_V2}" > token.txt image: name: hashicorp/terraform entrypoint: @@ -27,7 +25,9 @@ variables: FAST_SERVICE_ACCOUNT: ${service_account} FAST_WIF_PROVIDER: ${identity_provider} SSH_AUTH_SOCK: /tmp/ssh_agent.sock + %{~ if tf_providers_file != "" ~} TF_PROVIDERS_FILE: ${tf_providers_file} + %{~ endif ~} TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)} stages: @@ -40,13 +40,26 @@ cache: key: gcp-auth paths: - cicd-sa-credentials.json - - .tf-setup + - token.txt + %{~ if tf_providers_file != "" ~} + - ${tf_providers_file} + %{~ endif ~} + %{~ for f in tf_var_files ~} + - ${f} + %{~ endfor ~} gcp-auth: + id_tokens: + GITLAB_TOKEN: + aud: + %{~ for aud in audiences ~} + - ${aud} + %{~ endfor ~} image: name: google/cloud-sdk:slim stage: gcp-auth script: + - echo "$${GITLAB_TOKEN}" > token.txt - | gcloud iam workload-identity-pools create-cred-config \ $${FAST_WIF_PROVIDER} \ @@ -54,6 +67,7 @@ gcp-auth: --service-account-token-lifetime-seconds=3600 \ --output-file=$${GOOGLE_CREDENTIALS} \ --credential-source-file=token.txt + tf-files: dependencies: - gcp-auth @@ -63,15 +77,18 @@ tf-files: script: # - gcloud components install -q alpha - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS} - - mkdir -p .tf-setup - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/providers/$${TF_PROVIDERS_FILE}" .tf-setup/ - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/tfvars" .tf-setup/ + %{~ if tf_providers_file != "" ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./ + %{~ endif ~} + %{~ for f in tf_var_files ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./ + %{~ endfor ~} + - ls -l tf-plan: + dependencies: + - tf-files + stage: tf-plan # uncomment the following lines and set the SSH key secret for private modules repo # before_script: # - | @@ -80,20 +97,15 @@ tf-plan: # mkdir -p ~/.ssh # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts - stage: tf-plan script: - - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - - | - for f in $${TF_VAR_FILES}; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform plan - dependencies: - - tf-files tf-apply: + dependencies: + - tf-files + stage: tf-apply # uncomment the following lines and set the SSH key secret for private modules repo # before_script: # - | @@ -102,18 +114,10 @@ tf-apply: # mkdir -p ~/.ssh # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts - stage: tf-apply script: - - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - - | - for f in $${TF_VAR_FILES}; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform apply -input=false -auto-approve - dependencies: - - tf-files when: manual only: variables: diff --git a/fast/stages-multitenant/0-bootstrap-tenant/README.md b/fast/stages-multitenant/0-bootstrap-tenant/README.md index a10b808d..f831f9af 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/README.md +++ b/fast/stages-multitenant/0-bootstrap-tenant/README.md @@ -175,7 +175,6 @@ This configuration is possible but unsupported and only exists for development p - ## Files | name | description | modules | resources | @@ -197,35 +196,34 @@ This configuration is possible but unsupported and only exists for development p | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [automation](variables.tf#L20) | Automation resources created by the organization-level bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | -| [billing_account](variables.tf#L38) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | object({…}) | ✓ | | | -| [organization](variables.tf#L191) | Organization details. | object({…}) | ✓ | | 0-bootstrap | -| [prefix](variables.tf#L207) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | -| [tag_keys](variables.tf#L230) | Organization tag keys. | object({…}) | ✓ | | 1-resman | -| [tag_names](variables.tf#L241) | Customized names for resource management tags. | object({…}) | ✓ | | 1-resman | -| [tag_values](variables.tf#L252) | Organization resource management tag values. | map(string) | ✓ | | 1-resman | -| [tenant_config](variables.tf#L259) | Tenant configuration. Short name must be 4 characters or less. If `short_name_is_prefix` is true, short name must be 9 characters or less, and will be used as the prefix as is. | object({…}) | ✓ | | | -| [cicd_repositories](variables.tf#L48) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | -| [custom_roles](variables.tf#L94) | Custom roles defined at the organization level, in key => id format. | object({…}) | | null | 0-bootstrap | -| [fast_features](variables.tf#L104) | Selective control for top-level FAST features. | object({…}) | | {} | 0-bootstrap | -| [federated_identity_providers](variables.tf#L118) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | -| [group_iam](variables.tf#L132) | Tenant-level custom group IAM settings in group => [roles] format. | map(list(string)) | | {} | | -| [iam](variables.tf#L138) | Tenant-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | -| [iam_additive](variables.tf#L144) | Tenant-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | -| [locations](variables.tf#L150) | Optional locations for GCS, BigQuery, and logging buckets created here. These are the defaults set at the organization level, and can be overridden via the tenant config variable. | object({…}) | | {…} | 0-bootstrap | -| [log_sinks](variables.tf#L170) | Tenant-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | -| [outputs_location](variables.tf#L201) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | -| [project_parent_ids](variables.tf#L217) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the tenant folder as parent. | object({…}) | | {…} | | -| [test_principal](variables.tf#L300) | Used when testing to bypass the data source returning the current identity. | string | | null | | +| [automation](variables.tf#L20) | Automation resources created by the organization-level bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | +| [billing_account](variables.tf#L39) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | object({…}) | ✓ | | | +| [organization](variables.tf#L192) | Organization details. | object({…}) | ✓ | | 0-bootstrap | +| [prefix](variables.tf#L208) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | +| [tag_keys](variables.tf#L231) | Organization tag keys. | object({…}) | ✓ | | 1-resman | +| [tag_names](variables.tf#L242) | Customized names for resource management tags. | object({…}) | ✓ | | 1-resman | +| [tag_values](variables.tf#L253) | Organization resource management tag values. | map(string) | ✓ | | 1-resman | +| [tenant_config](variables.tf#L260) | Tenant configuration. Short name must be 4 characters or less. If `short_name_is_prefix` is true, short name must be 9 characters or less, and will be used as the prefix as is. | object({…}) | ✓ | | | +| [cicd_repositories](variables.tf#L49) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | +| [custom_roles](variables.tf#L95) | Custom roles defined at the organization level, in key => id format. | object({…}) | | null | 0-bootstrap | +| [fast_features](variables.tf#L105) | Selective control for top-level FAST features. | object({…}) | | {} | 0-bootstrap | +| [federated_identity_providers](variables.tf#L119) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | +| [group_iam](variables.tf#L133) | Tenant-level custom group IAM settings in group => [roles] format. | map(list(string)) | | {} | | +| [iam](variables.tf#L139) | Tenant-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | +| [iam_additive](variables.tf#L145) | Tenant-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | +| [locations](variables.tf#L151) | Optional locations for GCS, BigQuery, and logging buckets created here. These are the defaults set at the organization level, and can be overridden via the tenant config variable. | object({…}) | | {…} | 0-bootstrap | +| [log_sinks](variables.tf#L171) | Tenant-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | +| [outputs_location](variables.tf#L202) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | +| [project_parent_ids](variables.tf#L218) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the tenant folder as parent. | object({…}) | | {…} | | +| [test_principal](variables.tf#L301) | Used when testing to bypass the data source returning the current identity. | string | | null | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [cicd_workflows](outputs.tf#L107) | CI/CD workflows for tenant bootstrap and resource management stages. | ✓ | | -| [federated_identity](outputs.tf#L113) | Workload Identity Federation pool and providers. | | | -| [provider](outputs.tf#L123) | Terraform provider file for tenant resource management stage. | ✓ | stage-01 | -| [tenant_resources](outputs.tf#L130) | Tenant-level resources. | | | -| [tfvars](outputs.tf#L141) | Terraform variable files for the following tenant stages. | ✓ | | - +| [cicd_workflows](outputs.tf#L109) | CI/CD workflows for tenant bootstrap and resource management stages. | ✓ | | +| [federated_identity](outputs.tf#L115) | Workload Identity Federation pool and providers. | | | +| [provider](outputs.tf#L125) | Terraform provider file for tenant resource management stage. | ✓ | stage-01 | +| [tenant_resources](outputs.tf#L132) | Tenant-level resources. | | | +| [tfvars](outputs.tf#L143) | Terraform variable files for the following tenant stages. | ✓ | | diff --git a/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf b/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf index a25215af..fd0ec167 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf @@ -20,23 +20,27 @@ locals { _file_prefix = "tenants/${var.tenant_config.short_name}" # derive identity pool names from identity providers for easy reference cicd_identity_pools = { - for k, v in local.cicd_identity_providers : + for k, v in local.cicd_providers : k => split("/providers/", v.name)[0] } # merge org-level and tenant-level identity providers - cicd_identity_providers = merge( + cicd_providers = merge( var.automation.federated_identity_providers, { for k, v in google_iam_workload_identity_pool_provider.default : k => { + audiences = concat( + v.oidc[0].allowed_audiences, + ["https://iam.googleapis.com/${v.name}"] + ) issuer = local.identity_providers[k].issuer - issuer_uri = local.identity_providers[k].issuer_uri + issuer_uri = try(v.oidc[0].issuer_uri, null) name = v.name principal_tpl = local.identity_providers[k].principal_tpl principalset_tpl = local.identity_providers[k].principalset_tpl } - }) - # filter CI/CD repositories to only keep valid ones + } + ) cicd_repositories = { for k, v in coalesce(var.cicd_repositories, {}) : k => v if( @@ -46,7 +50,7 @@ locals { try(v.type, null) == "sourcerepo" || contains( - keys(local.cicd_identity_providers), + keys(local.cicd_providers), coalesce(try(v.identity_provider, null), ":") ) ) @@ -111,12 +115,12 @@ module "automation-tf-cicd-sa-bootstrap" { "roles/iam.workloadIdentityUser" = [ each.value.branch == null ? format( - local.cicd_identity_providers[each.value.identity_provider].principalset_tpl, + local.cicd_providers[each.value.identity_provider].principalset_tpl, local.cicd_identity_pools[each.value.identity_provider], each.value.name ) : format( - local.cicd_identity_providers[each.value.identity_provider].principal_tpl, + local.cicd_providers[each.value.identity_provider].principal_tpl, local.cicd_identity_pools[each.value.identity_provider], each.value.name, each.value.branch @@ -201,12 +205,12 @@ module "automation-tf-cicd-sa-resman" { "roles/iam.workloadIdentityUser" = [ each.value.branch == null ? format( - local.cicd_identity_providers[each.value.identity_provider].principalset_tpl, + local.cicd_providers[each.value.identity_provider].principalset_tpl, local.cicd_identity_pools[each.value.identity_provider], each.value.name ) : format( - local.cicd_identity_providers[each.value.identity_provider].principal_tpl, + local.cicd_providers[each.value.identity_provider].principal_tpl, local.cicd_identity_pools[each.value.identity_provider], each.value.name, each.value.branch diff --git a/fast/stages-multitenant/0-bootstrap-tenant/identity-providers.tf b/fast/stages-multitenant/0-bootstrap-tenant/identity-providers.tf index 3f8499b7..085a30f0 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/identity-providers.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/identity-providers.tf @@ -38,7 +38,7 @@ locals { principal_tpl = "principal://iam.googleapis.com/%s/subject/repo:%s:ref:refs/heads/%s" principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s" } - # https://docs.gitlab.com/ee/ci/cloud_services/index.html#how-it-works + # https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#token-payload gitlab = { attribute_mapping = { "google.subject" = "assertion.sub" @@ -56,10 +56,9 @@ locals { "attribute.ref_protected" = "assertion.ref_protected" "attribute.ref_type" = "assertion.ref_type" } - allowed_audiences = ["https://gitlab.com"] - issuer_uri = "https://gitlab.com" - principal_tpl = "principalSet://iam.googleapis.com/%s/attribute.sub/project_path:%s:ref_type:branch:ref:%s" - principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s" + issuer_uri = "https://gitlab.com" + principal_tpl = "principalSet://iam.googleapis.com/%s/attribute.sub/project_path:%s:ref_type:branch:ref:%s" + principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s" } } } @@ -82,13 +81,11 @@ resource "google_iam_workload_identity_pool_provider" "default" { attribute_condition = each.value.attribute_condition attribute_mapping = each.value.attribute_mapping oidc { - allowed_audiences = ( - try(each.value.custom_settings.allowed_audiences, null) != null - ? each.value.custom_settings.allowed_audiences - : try(each.value.allowed_audiences, null) - ) + # Setting an empty list configures allowed_audiences to the url of the provider + allowed_audiences = each.value.custom_settings.audiences + # If users don't provide an issuer_uri, we set the public one for the plaform choosed. issuer_uri = ( - try(each.value.custom_settings.issuer_uri, null) != null + each.value.custom_settings.issuer_uri != null ? each.value.custom_settings.issuer_uri : try(each.value.issuer_uri, null) ) diff --git a/fast/stages-multitenant/0-bootstrap-tenant/outputs.tf b/fast/stages-multitenant/0-bootstrap-tenant/outputs.tf index c61c584e..883acac9 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/outputs.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/outputs.tf @@ -20,8 +20,9 @@ locals { "${path.module}/templates/workflow-${v.type}.yaml", ( k == "bootstrap" ? { + audiences = local.cicd_providers[v["identity_provider"]].audiences identity_provider = try( - local.cicd_identity_providers[v["identity_provider"]].name, "" + local.cicd_providers[v["identity_provider"]].name, "" ) outputs_bucket = var.automation.outputs_bucket service_account = try( @@ -36,8 +37,9 @@ locals { ] } : { + audiences = local.cicd_providers[v["identity_provider"]].audiences identity_provider = try( - local.cicd_identity_providers[v["identity_provider"]].name, "" + local.cicd_providers[v["identity_provider"]].name, "" ) outputs_bucket = module.automation-tf-output-gcs.name service_account = try( @@ -70,7 +72,7 @@ locals { try(google_iam_workload_identity_pool.default.0.name, null), var.automation.federated_identity_pool, ]) - federated_identity_providers = local.cicd_identity_providers + federated_identity_providers = local.cicd_providers service_accounts = merge( { resman = module.automation-tf-resman-sa.email }, { @@ -116,7 +118,7 @@ output "federated_identity" { pool = try( google_iam_workload_identity_pool.default.0.name, null ) - providers = local.cicd_identity_providers + providers = local.cicd_providers } } diff --git a/fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-gitlab.yaml b/fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-gitlab.yaml index 739e7485..13057e11 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-gitlab.yaml +++ b/fast/stages-multitenant/0-bootstrap-tenant/templates/workflow-gitlab.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +13,6 @@ # limitations under the License. default: - before_script: - - echo "$${CI_JOB_JWT_V2}" > token.txt image: name: hashicorp/terraform entrypoint: @@ -42,13 +40,26 @@ cache: key: gcp-auth paths: - cicd-sa-credentials.json - - .tf-setup + - token.txt + %{~ if tf_providers_file != "" ~} + - ${tf_providers_file} + %{~ endif ~} + %{~ for f in tf_var_files ~} + - ${f} + %{~ endfor ~} gcp-auth: + id_tokens: + GITLAB_TOKEN: + aud: + %{~ for aud in audiences ~} + - ${aud} + %{~ endfor ~} image: name: google/cloud-sdk:slim stage: gcp-auth script: + - echo "$${GITLAB_TOKEN}" > token.txt - | gcloud iam workload-identity-pools create-cred-config \ $${FAST_WIF_PROVIDER} \ @@ -56,6 +67,7 @@ gcp-auth: --service-account-token-lifetime-seconds=3600 \ --output-file=$${GOOGLE_CREDENTIALS} \ --credential-source-file=token.txt + tf-files: dependencies: - gcp-auth @@ -65,17 +77,18 @@ tf-files: script: # - gcloud components install -q alpha - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS} - - mkdir -p .tf-setup %{~ if tf_providers_file != "" ~} - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/providers/$${TF_PROVIDERS_FILE}" .tf-setup/ + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./ %{~ endif ~} - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/tfvars" .tf-setup/ + %{~ for f in tf_var_files ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./ + %{~ endfor ~} + - ls -l tf-plan: + dependencies: + - tf-files + stage: tf-plan # uncomment the following lines and set the SSH key secret for private modules repo # before_script: # - | @@ -84,20 +97,15 @@ tf-plan: # mkdir -p ~/.ssh # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts - stage: tf-plan script: - - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - - | - for f in $${TF_VAR_FILES}; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform plan - dependencies: - - tf-files tf-apply: + dependencies: + - tf-files + stage: tf-apply # uncomment the following lines and set the SSH key secret for private modules repo # before_script: # - | @@ -106,18 +114,10 @@ tf-apply: # mkdir -p ~/.ssh # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts - stage: tf-apply script: - - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - - | - for f in $${TF_VAR_FILES}; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform apply -input=false -auto-approve - dependencies: - - tf-files when: manual only: variables: diff --git a/fast/stages-multitenant/0-bootstrap-tenant/variables.tf b/fast/stages-multitenant/0-bootstrap-tenant/variables.tf index 718818ea..af56db39 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/variables.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/variables.tf @@ -26,6 +26,7 @@ variable "automation" { project_number = string federated_identity_pool = string federated_identity_providers = map(object({ + audiences = list(string) issuer = string issuer_uri = string name = string @@ -118,12 +119,12 @@ variable "fast_features" { variable "federated_identity_providers" { description = "Workload Identity Federation pools. The `cicd_repositories` variable references keys here." type = map(object({ - attribute_condition = string + attribute_condition = optional(string) issuer = string - custom_settings = object({ - issuer_uri = string - allowed_audiences = list(string) - }) + custom_settings = optional(object({ + issuer_uri = optional(string) + audiences = optional(list(string), []) + }), {}) })) default = {} nullable = false diff --git a/fast/stages-multitenant/1-resman-tenant/README.md b/fast/stages-multitenant/1-resman-tenant/README.md index 9e1188bb..c38f173f 100644 --- a/fast/stages-multitenant/1-resman-tenant/README.md +++ b/fast/stages-multitenant/1-resman-tenant/README.md @@ -126,7 +126,6 @@ Once the configuration is done just go through the usual `init/apply` cycle. On - ## Files | name | description | modules | resources | @@ -154,21 +153,21 @@ Once the configuration is done just go through the usual `init/apply` cycle. On | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | -| [billing_account](variables.tf#L51) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | object({…}) | ✓ | | 0-bootstrap | -| [organization](variables.tf#L204) | Organization details. | object({…}) | ✓ | | 0-bootstrap | -| [prefix](variables.tf#L226) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | +| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | +| [billing_account](variables.tf#L52) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | object({…}) | ✓ | | 0-bootstrap | +| [organization](variables.tf#L205) | Organization details. | object({…}) | ✓ | | 0-bootstrap | +| [prefix](variables.tf#L227) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | | [root_node](variables.tf#L237) | Root folder node for the tenant, in folders/nnnnnn format. | string | ✓ | | | | [short_name](variables.tf#L242) | Short name used to identify the tenant. | string | ✓ | | | | [tags](variables.tf#L247) | Resource management tags. | object({…}) | ✓ | | | -| [cicd_repositories](variables.tf#L62) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | -| [custom_roles](variables.tf#L144) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 0-bootstrap | -| [data_dir](variables.tf#L153) | Relative path for the folder storing configuration data. | string | | "data" | | -| [fast_features](variables.tf#L159) | Selective control for top-level FAST features. | object({…}) | | {} | 0-0-bootstrap | -| [groups](variables.tf#L173) | Group names to grant organization-level permissions. | object({…}) | | {} | 0-bootstrap | -| [locations](variables.tf#L186) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | 0-bootstrap | -| [organization_policy_data_path](variables.tf#L214) | Path for the data folder used by the organization policies factory. | string | | null | | -| [outputs_location](variables.tf#L220) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | +| [cicd_repositories](variables.tf#L63) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | +| [custom_roles](variables.tf#L145) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 0-bootstrap | +| [data_dir](variables.tf#L154) | Relative path for the folder storing configuration data. | string | | "data" | | +| [fast_features](variables.tf#L160) | Selective control for top-level FAST features. | object({…}) | | {} | 0-0-bootstrap | +| [groups](variables.tf#L174) | Group names to grant organization-level permissions. | object({…}) | | {} | 0-bootstrap | +| [locations](variables.tf#L187) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | 0-bootstrap | +| [organization_policy_data_path](variables.tf#L215) | Path for the data folder used by the organization policies factory. | string | | null | | +| [outputs_location](variables.tf#L221) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | | [team_folders](variables.tf#L265) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | | | [test_skip_data_sources](variables.tf#L275) | Used when testing to bypass data sources. | bool | | false | | @@ -176,15 +175,14 @@ Once the configuration is done just go through the usual `init/apply` cycle. On | name | description | sensitive | consumers | |---|---|:---:|---| -| [cicd_repositories](outputs.tf#L189) | WIF configuration for CI/CD repositories. | | | -| [dataplatform](outputs.tf#L203) | Data for the Data Platform stage. | | | -| [gke_multitenant](outputs.tf#L219) | Data for the GKE multitenant stage. | | 03-gke-multitenant | -| [networking](outputs.tf#L240) | Data for the networking stage. | | | -| [project_factories](outputs.tf#L249) | Data for the project factories stage. | | | -| [providers](outputs.tf#L264) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | -| [sandbox](outputs.tf#L271) | Data for the sandbox stage. | | xx-sandbox | -| [security](outputs.tf#L285) | Data for the networking stage. | | 02-security | -| [teams](outputs.tf#L295) | Data for the teams stage. | | | -| [tfvars](outputs.tf#L307) | Terraform variable files for the following stages. | ✓ | | - +| [cicd_repositories](outputs.tf#L192) | WIF configuration for CI/CD repositories. | | | +| [dataplatform](outputs.tf#L206) | Data for the Data Platform stage. | | | +| [gke_multitenant](outputs.tf#L222) | Data for the GKE multitenant stage. | | 03-gke-multitenant | +| [networking](outputs.tf#L243) | Data for the networking stage. | | | +| [project_factories](outputs.tf#L252) | Data for the project factories stage. | | | +| [providers](outputs.tf#L267) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | +| [sandbox](outputs.tf#L274) | Data for the sandbox stage. | | xx-sandbox | +| [security](outputs.tf#L288) | Data for the networking stage. | | 02-security | +| [teams](outputs.tf#L298) | Data for the teams stage. | | | +| [tfvars](outputs.tf#L310) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages-multitenant/1-resman-tenant/outputs.tf b/fast/stages-multitenant/1-resman-tenant/outputs.tf index 592f995e..dadfb057 100644 --- a/fast/stages-multitenant/1-resman-tenant/outputs.tf +++ b/fast/stages-multitenant/1-resman-tenant/outputs.tf @@ -62,6 +62,9 @@ locals { for k, v in local.cicd_repositories : k => templatefile( "${path.module}/templates/workflow-${v.type}.yaml", merge(local.cicd_workflow_attrs[k], { + audiences = try( + local.cicd_identity_providers[v.identity_provider].audiences, null + ) identity_provider = try( local.cicd_identity_providers[v.identity_provider].name, null ) diff --git a/fast/stages-multitenant/1-resman-tenant/templates/workflow-gitlab.yaml b/fast/stages-multitenant/1-resman-tenant/templates/workflow-gitlab.yaml index 8981e70b..13057e11 100644 --- a/fast/stages-multitenant/1-resman-tenant/templates/workflow-gitlab.yaml +++ b/fast/stages-multitenant/1-resman-tenant/templates/workflow-gitlab.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +13,6 @@ # limitations under the License. default: - before_script: - - echo "$${CI_JOB_JWT_V2}" > token.txt image: name: hashicorp/terraform entrypoint: @@ -27,7 +25,9 @@ variables: FAST_SERVICE_ACCOUNT: ${service_account} FAST_WIF_PROVIDER: ${identity_provider} SSH_AUTH_SOCK: /tmp/ssh_agent.sock + %{~ if tf_providers_file != "" ~} TF_PROVIDERS_FILE: ${tf_providers_file} + %{~ endif ~} TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)} stages: @@ -40,13 +40,26 @@ cache: key: gcp-auth paths: - cicd-sa-credentials.json - - .tf-setup + - token.txt + %{~ if tf_providers_file != "" ~} + - ${tf_providers_file} + %{~ endif ~} + %{~ for f in tf_var_files ~} + - ${f} + %{~ endfor ~} gcp-auth: + id_tokens: + GITLAB_TOKEN: + aud: + %{~ for aud in audiences ~} + - ${aud} + %{~ endfor ~} image: name: google/cloud-sdk:slim stage: gcp-auth script: + - echo "$${GITLAB_TOKEN}" > token.txt - | gcloud iam workload-identity-pools create-cred-config \ $${FAST_WIF_PROVIDER} \ @@ -54,6 +67,7 @@ gcp-auth: --service-account-token-lifetime-seconds=3600 \ --output-file=$${GOOGLE_CREDENTIALS} \ --credential-source-file=token.txt + tf-files: dependencies: - gcp-auth @@ -63,15 +77,18 @@ tf-files: script: # - gcloud components install -q alpha - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS} - - mkdir -p .tf-setup - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/providers/$${TF_PROVIDERS_FILE}" .tf-setup/ - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/tfvars" .tf-setup/ + %{~ if tf_providers_file != "" ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./ + %{~ endif ~} + %{~ for f in tf_var_files ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./ + %{~ endfor ~} + - ls -l tf-plan: + dependencies: + - tf-files + stage: tf-plan # uncomment the following lines and set the SSH key secret for private modules repo # before_script: # - | @@ -80,20 +97,15 @@ tf-plan: # mkdir -p ~/.ssh # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts - stage: tf-plan script: - - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - - | - for f in $${TF_VAR_FILES}; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform plan - dependencies: - - tf-files tf-apply: + dependencies: + - tf-files + stage: tf-apply # uncomment the following lines and set the SSH key secret for private modules repo # before_script: # - | @@ -102,18 +114,10 @@ tf-apply: # mkdir -p ~/.ssh # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts - stage: tf-apply script: - - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - - | - for f in $${TF_VAR_FILES}; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform apply -input=false -auto-approve - dependencies: - - tf-files when: manual only: variables: diff --git a/fast/stages-multitenant/1-resman-tenant/variables.tf b/fast/stages-multitenant/1-resman-tenant/variables.tf index 1c399be2..bcda7fd6 100644 --- a/fast/stages-multitenant/1-resman-tenant/variables.tf +++ b/fast/stages-multitenant/1-resman-tenant/variables.tf @@ -26,6 +26,7 @@ variable "automation" { project_number = string federated_identity_pools = list(string) federated_identity_providers = map(object({ + audiences = list(string) issuer = string issuer_uri = string name = string @@ -227,7 +228,6 @@ variable "prefix" { # tfdoc:variable:source 0-bootstrap description = "Prefix used for resources that need unique names. Use 9 characters or less." type = string - validation { condition = try(length(var.prefix), 0) < 13 error_message = "Use a maximum of 12 characters for prefix (which is a combination of org prefix and tenant short name)." diff --git a/fast/stages/0-bootstrap/README.md b/fast/stages/0-bootstrap/README.md index 105b044f..a9a28b1b 100644 --- a/fast/stages/0-bootstrap/README.md +++ b/fast/stages/0-bootstrap/README.md @@ -209,10 +209,10 @@ Then make sure you have configured the correct values for the following variable - `organization.id`, `organization.domain`, `organization.customer_id` the id, domain and customer id of your organization, derived from the Cloud Console UI or by running `gcloud organizations list` - `prefix` - the fixed org-level prefix used in your naming, maximum 9 characters long. Note that if you are using multitenant stages, then you will later need to configure a `tenant prefix`. - This `tenant prefix` can have a maximum length of 2 characters, - plus any unused characters from the from the `prefix`. - For example, if you specify a `prefix` that is 7 characters long, + the fixed org-level prefix used in your naming, maximum 9 characters long. Note that if you are using multitenant stages, then you will later need to configure a `tenant prefix`. + This `tenant prefix` can have a maximum length of 2 characters, + plus any unused characters from the from the `prefix`. + For example, if you specify a `prefix` that is 7 characters long, then your `tenant prefix` can have a maximum of 4 characters. You can also adapt the example that follows to your needs: @@ -421,7 +421,7 @@ federated_identity_providers = { attribute_condition = "attribute.namespace_path==\"my-gitlab-org\"" issuer = "gitlab" custom_settings = { - allowed_audiences = ["https://gitlab.fast.example.com"] + audiences = ["https://gitlab.fast.example.com"] issuer_uri = "https://gitlab.fast.example.com" } } @@ -516,7 +516,7 @@ The remaining configuration is manual, as it regards the repositories themselves | [custom_role_names](variables.tf#L79) | Names of custom roles defined at the org level. | object({…}) | | {…} | | | [custom_roles](variables.tf#L93) | Map of role names => list of permissions to additionally create at the organization level. | map(list(string)) | | {} | | | [fast_features](variables.tf#L100) | Selective control for top-level FAST features. | object({…}) | | {} | | -| [federated_identity_providers](variables.tf#L113) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | +| [federated_identity_providers](variables.tf#L113) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | | [groups](variables.tf#L132) | Group names or emails to grant organization-level permissions. If just the name is provided, the default organization domain is assumed. | map(string) | | {…} | | | [iam](variables.tf#L150) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | | [iam_additive](variables.tf#L156) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | diff --git a/fast/stages/0-bootstrap/cicd.tf b/fast/stages/0-bootstrap/cicd.tf index f0b321d9..96839ec3 100644 --- a/fast/stages/0-bootstrap/cicd.tf +++ b/fast/stages/0-bootstrap/cicd.tf @@ -20,9 +20,9 @@ locals { cicd_providers = { for k, v in google_iam_workload_identity_pool_provider.default : k => { - audience = try( - v.oidc[0].allowed_audiences[0], - "https://iam.googleapis.com/${v.name}" + audiences = concat( + v.oidc[0].allowed_audiences, + ["https://iam.googleapis.com/${v.name}"] ) issuer = local.identity_providers[k].issuer issuer_uri = try(v.oidc[0].issuer_uri, null) @@ -39,10 +39,15 @@ locals { ( try(v.type, null) == "sourcerepo" || - contains(keys(local.identity_providers), coalesce(try(v.identity_provider, null), ":")) + contains( + keys(local.identity_providers), + coalesce(try(v.identity_provider, null), ":") + ) ) && - fileexists(format("${path.module}/templates/workflow-%s.yaml", try(v.type, ""))) + fileexists( + format("${path.module}/templates/workflow-%s.yaml", try(v.type, "")) + ) ) } cicd_workflow_providers = { diff --git a/fast/stages/0-bootstrap/identity-providers.tf b/fast/stages/0-bootstrap/identity-providers.tf index 1c1c8b20..085a30f0 100644 --- a/fast/stages/0-bootstrap/identity-providers.tf +++ b/fast/stages/0-bootstrap/identity-providers.tf @@ -82,7 +82,7 @@ resource "google_iam_workload_identity_pool_provider" "default" { attribute_mapping = each.value.attribute_mapping oidc { # Setting an empty list configures allowed_audiences to the url of the provider - allowed_audiences = each.value.custom_settings.allowed_audiences + allowed_audiences = each.value.custom_settings.audiences # If users don't provide an issuer_uri, we set the public one for the plaform choosed. issuer_uri = ( each.value.custom_settings.issuer_uri != null diff --git a/fast/stages/0-bootstrap/outputs.tf b/fast/stages/0-bootstrap/outputs.tf index 1db70bc4..ea980fca 100644 --- a/fast/stages/0-bootstrap/outputs.tf +++ b/fast/stages/0-bootstrap/outputs.tf @@ -22,7 +22,7 @@ locals { "${path.module}/templates/workflow-${v.type}.yaml", { # If users give a list of custom audiences we set by default the first element. # If no audiences are given, we set https://iam.googleapis.com/{PROVIDER_NAME} - audience = local.cicd_providers[v["identity_provider"]].audience + audiences = local.cicd_providers[v["identity_provider"]].audiences identity_provider = try( local.cicd_providers[v["identity_provider"]].name, "" ) diff --git a/fast/stages/0-bootstrap/templates/workflow-gitlab.yaml b/fast/stages/0-bootstrap/templates/workflow-gitlab.yaml index 33e55d21..13057e11 100644 --- a/fast/stages/0-bootstrap/templates/workflow-gitlab.yaml +++ b/fast/stages/0-bootstrap/templates/workflow-gitlab.yaml @@ -20,13 +20,14 @@ default: - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" variables: - AUDIENCE: ${audience} GOOGLE_CREDENTIALS: cicd-sa-credentials.json FAST_OUTPUTS_BUCKET: ${outputs_bucket} FAST_SERVICE_ACCOUNT: ${service_account} FAST_WIF_PROVIDER: ${identity_provider} SSH_AUTH_SOCK: /tmp/ssh_agent.sock + %{~ if tf_providers_file != "" ~} TF_PROVIDERS_FILE: ${tf_providers_file} + %{~ endif ~} TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)} stages: @@ -38,16 +39,27 @@ stages: cache: key: gcp-auth paths: - - .tf-setup + - cicd-sa-credentials.json + - token.txt + %{~ if tf_providers_file != "" ~} + - ${tf_providers_file} + %{~ endif ~} + %{~ for f in tf_var_files ~} + - ${f} + %{~ endfor ~} gcp-auth: - stage: gcp-auth id_tokens: GITLAB_TOKEN: - aud: "$${AUDIENCE}" + aud: + %{~ for aud in audiences ~} + - ${aud} + %{~ endfor ~} image: name: google/cloud-sdk:slim + stage: gcp-auth script: + - echo "$${GITLAB_TOKEN}" > token.txt - | gcloud iam workload-identity-pools create-cred-config \ $${FAST_WIF_PROVIDER} \ @@ -55,30 +67,27 @@ gcp-auth: --service-account-token-lifetime-seconds=3600 \ --output-file=$${GOOGLE_CREDENTIALS} \ --credential-source-file=token.txt - - rm token.txt - artifacts: - untracked: true tf-files: - stage: tf-files + dependencies: + - gcp-auth image: name: google/cloud-sdk:slim + stage: tf-files script: # - gcloud components install -q alpha - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS} - - mkdir -p .tf-setup - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/providers/$${TF_PROVIDERS_FILE}" .tf-setup/ - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/tfvars" .tf-setup/ - artifacts: - untracked: true - dependencies: - - gcp-auth + %{~ if tf_providers_file != "" ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./ + %{~ endif ~} + %{~ for f in tf_var_files ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./ + %{~ endfor ~} + - ls -l tf-plan: + dependencies: + - tf-files stage: tf-plan # uncomment the following lines and set the SSH key secret for private modules repo # before_script: @@ -89,20 +98,13 @@ tf-plan: # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts script: - - cp ".tf-setup/$${TF_PROVIDERS_FILE}" ./ - - | - for f in "$${TF_VAR_FILES}"; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform plan - artifacts: - untracked: true - dependencies: - - tf-files tf-apply: + dependencies: + - tf-files stage: tf-apply # uncomment the following lines and set the SSH key secret for private modules repo # before_script: @@ -113,18 +115,9 @@ tf-apply: # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts script: - - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - - | - for f in $${TF_VAR_FILES}; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform apply -input=false -auto-approve - artifacts: - untracked: true - dependencies: - - tf-files when: manual only: variables: diff --git a/fast/stages/0-bootstrap/variables.tf b/fast/stages/0-bootstrap/variables.tf index f41d6dfa..1008c240 100644 --- a/fast/stages/0-bootstrap/variables.tf +++ b/fast/stages/0-bootstrap/variables.tf @@ -116,8 +116,8 @@ variable "federated_identity_providers" { attribute_condition = optional(string) issuer = string custom_settings = optional(object({ - issuer_uri = optional(string) - allowed_audiences = optional(list(string), []) + issuer_uri = optional(string) + audiences = optional(list(string), []) }), {}) })) default = {} diff --git a/fast/stages/1-resman/README.md b/fast/stages/1-resman/README.md index 4dec2945..d938020f 100644 --- a/fast/stages/1-resman/README.md +++ b/fast/stages/1-resman/README.md @@ -365,7 +365,7 @@ Due to its simplicity, this stage lends itself easily to customizations: adding | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | +| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | | [billing_account](variables.tf#L39) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | object({…}) | ✓ | | 0-bootstrap | | [organization](variables.tf#L192) | Organization details. | object({…}) | ✓ | | 0-bootstrap | | [prefix](variables.tf#L216) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | diff --git a/fast/stages/1-resman/outputs.tf b/fast/stages/1-resman/outputs.tf index 552d42d7..41994e98 100644 --- a/fast/stages/1-resman/outputs.tf +++ b/fast/stages/1-resman/outputs.tf @@ -62,8 +62,8 @@ locals { for k, v in local.cicd_repositories : k => templatefile( "${path.module}/templates/workflow-${v.type}.yaml", merge(local.cicd_workflow_attrs[k], { - audience = try( - local.identity_providers[v.identity_provider].audience, null + audiences = try( + local.identity_providers[v.identity_provider].audiences, null ) identity_provider = try( local.identity_providers[v.identity_provider].name, null diff --git a/fast/stages/1-resman/templates/workflow-gitlab.yaml b/fast/stages/1-resman/templates/workflow-gitlab.yaml index 33e55d21..13057e11 100644 --- a/fast/stages/1-resman/templates/workflow-gitlab.yaml +++ b/fast/stages/1-resman/templates/workflow-gitlab.yaml @@ -20,13 +20,14 @@ default: - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" variables: - AUDIENCE: ${audience} GOOGLE_CREDENTIALS: cicd-sa-credentials.json FAST_OUTPUTS_BUCKET: ${outputs_bucket} FAST_SERVICE_ACCOUNT: ${service_account} FAST_WIF_PROVIDER: ${identity_provider} SSH_AUTH_SOCK: /tmp/ssh_agent.sock + %{~ if tf_providers_file != "" ~} TF_PROVIDERS_FILE: ${tf_providers_file} + %{~ endif ~} TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)} stages: @@ -38,16 +39,27 @@ stages: cache: key: gcp-auth paths: - - .tf-setup + - cicd-sa-credentials.json + - token.txt + %{~ if tf_providers_file != "" ~} + - ${tf_providers_file} + %{~ endif ~} + %{~ for f in tf_var_files ~} + - ${f} + %{~ endfor ~} gcp-auth: - stage: gcp-auth id_tokens: GITLAB_TOKEN: - aud: "$${AUDIENCE}" + aud: + %{~ for aud in audiences ~} + - ${aud} + %{~ endfor ~} image: name: google/cloud-sdk:slim + stage: gcp-auth script: + - echo "$${GITLAB_TOKEN}" > token.txt - | gcloud iam workload-identity-pools create-cred-config \ $${FAST_WIF_PROVIDER} \ @@ -55,30 +67,27 @@ gcp-auth: --service-account-token-lifetime-seconds=3600 \ --output-file=$${GOOGLE_CREDENTIALS} \ --credential-source-file=token.txt - - rm token.txt - artifacts: - untracked: true tf-files: - stage: tf-files + dependencies: + - gcp-auth image: name: google/cloud-sdk:slim + stage: tf-files script: # - gcloud components install -q alpha - gcloud config set auth/credential_file_override $${GOOGLE_CREDENTIALS} - - mkdir -p .tf-setup - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/providers/$${TF_PROVIDERS_FILE}" .tf-setup/ - - | - gcloud alpha storage cp -r \ - "gs://$${FAST_OUTPUTS_BUCKET}/tfvars" .tf-setup/ - artifacts: - untracked: true - dependencies: - - gcp-auth + %{~ if tf_providers_file != "" ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/providers/${tf_providers_file}" ./ + %{~ endif ~} + %{~ for f in tf_var_files ~} + - gcloud alpha storage cp -r "gs://$${FAST_OUTPUTS_BUCKET}/tfvars/${f}" ./ + %{~ endfor ~} + - ls -l tf-plan: + dependencies: + - tf-files stage: tf-plan # uncomment the following lines and set the SSH key secret for private modules repo # before_script: @@ -89,20 +98,13 @@ tf-plan: # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts script: - - cp ".tf-setup/$${TF_PROVIDERS_FILE}" ./ - - | - for f in "$${TF_VAR_FILES}"; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform plan - artifacts: - untracked: true - dependencies: - - tf-files tf-apply: + dependencies: + - tf-files stage: tf-apply # uncomment the following lines and set the SSH key secret for private modules repo # before_script: @@ -113,18 +115,9 @@ tf-apply: # ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts # ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts script: - - cp .tf-setup/$${TF_PROVIDERS_FILE} ./ - - | - for f in $${TF_VAR_FILES}; do - ln -s ".tf-setup/tfvars/$f" ./ - done - terraform init - terraform validate - terraform apply -input=false -auto-approve - artifacts: - untracked: true - dependencies: - - tf-files when: manual only: variables: diff --git a/fast/stages/1-resman/variables.tf b/fast/stages/1-resman/variables.tf index a613e5ad..f255423a 100644 --- a/fast/stages/1-resman/variables.tf +++ b/fast/stages/1-resman/variables.tf @@ -26,7 +26,7 @@ variable "automation" { project_number = string federated_identity_pool = string federated_identity_providers = map(object({ - audience = string + audiences = list(string) issuer = string issuer_uri = string name = string From 1644ab8279c681b92ffd2fdfd4ab11f58c2edb21 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 15 Aug 2023 15:19:29 +0200 Subject: [PATCH 10/46] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f05cd212..a8e5a07f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,12 @@ All notable changes to this project will be documented in this file. ### FAST +- [[#1593](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1593)] Fix FAST CI/CD for Gitlab ([ludoo](https://github.com/ludoo)) - [[#1583](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1583)] Fix module path for teams cicd ([ludoo](https://github.com/ludoo)) ### MODULES +- [[#1591](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1591)] feat: 🎸 (modules/cloudsql-instance):add project_id for ssl cert ([erabusi](https://github.com/erabusi)) - [[#1589](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1589)] Add new `iam_members` variable to IAM additive module interfaces ([ludoo](https://github.com/ludoo)) - [[#1588](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1588)] feat: 🎸 (modules/cloudsql-instance): enable require_ssl cert support ([erabusi](https://github.com/erabusi)) - [[#1587](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1587)] **incompatible change:** Fix factory rules key in net firewall policy module ([ludoo](https://github.com/ludoo)) From def2f476d1d6a65010053398dcffc16523c226e7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 15 Aug 2023 16:28:23 +0200 Subject: [PATCH 11/46] Add support for conditions to `iam_members` module variables (#1594) * project * data-catalog-policy-tag * dataproc * folder * iam-service-account * kms * net-vpc * organization * source-repository * dataplex-datascan --- modules/data-catalog-policy-tag/README.md | 12 ++-- modules/data-catalog-policy-tag/iam.tf | 8 +++ modules/data-catalog-policy-tag/variables.tf | 5 ++ modules/dataplex-datascan/README.md | 16 ++--- modules/dataplex-datascan/iam.tf | 8 +++ modules/dataplex-datascan/variables.tf | 5 ++ modules/dataproc/README.md | 14 ++-- modules/dataproc/iam.tf | 8 +++ modules/dataproc/variables.tf | 5 ++ modules/folder/README.md | 22 +++--- modules/folder/iam.tf | 8 +++ modules/folder/variables.tf | 5 ++ modules/iam-service-account/README.md | 20 +++--- modules/iam-service-account/iam.tf | 8 +++ modules/iam-service-account/variables.tf | 5 ++ modules/kms/README.md | 22 +++--- modules/kms/iam.tf | 16 +++++ modules/kms/variables.tf | 10 +++ modules/net-vpc/README.md | 10 +-- modules/net-vpc/subnets.tf | 8 +++ modules/net-vpc/variables.tf | 5 ++ modules/organization/README.md | 26 +++---- modules/organization/iam.tf | 8 +++ modules/organization/variables.tf | 5 ++ modules/project/README.md | 71 +++++++++++-------- modules/project/iam.tf | 8 +++ modules/project/variables.tf | 5 ++ modules/source-repository/README.md | 8 +-- modules/source-repository/iam.tf | 8 +++ modules/source-repository/variables.tf | 5 ++ .../modules/project/examples/iam-members.yaml | 14 +++- 31 files changed, 272 insertions(+), 106 deletions(-) diff --git a/modules/data-catalog-policy-tag/README.md b/modules/data-catalog-policy-tag/README.md index 6f53fcbd..5a1c2055 100644 --- a/modules/data-catalog-policy-tag/README.md +++ b/modules/data-catalog-policy-tag/README.md @@ -66,18 +66,18 @@ module "cmn-dc" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L69) | Name of this taxonomy. | string | ✓ | | -| [project_id](variables.tf#L84) | GCP project id. | | ✓ | | +| [name](variables.tf#L74) | Name of this taxonomy. | string | ✓ | | +| [project_id](variables.tf#L89) | GCP project id. | | ✓ | | | [activated_policy_types](variables.tf#L17) | A list of policy types that are activated for this taxonomy. | list(string) | | ["FINE_GRAINED_ACCESS_CONTROL"] | | [description](variables.tf#L23) | Description of this taxonomy. | string | | "Taxonomy - Terraform managed" | | [group_iam](variables.tf#L29) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L35) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L41) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L47) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L53) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [location](variables.tf#L63) | Data Catalog Taxonomy location. | string | | "eu" | -| [prefix](variables.tf#L74) | Optional prefix used to generate project id and name. | string | | null | -| [tags](variables.tf#L88) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | +| [iam_members](variables.tf#L53) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [location](variables.tf#L68) | Data Catalog Taxonomy location. | string | | "eu" | +| [prefix](variables.tf#L79) | Optional prefix used to generate project id and name. | string | | null | +| [tags](variables.tf#L93) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | ## Outputs diff --git a/modules/data-catalog-policy-tag/iam.tf b/modules/data-catalog-policy-tag/iam.tf index 6f79aebf..9e7f3a09 100644 --- a/modules/data-catalog-policy-tag/iam.tf +++ b/modules/data-catalog-policy-tag/iam.tf @@ -80,6 +80,14 @@ resource "google_data_catalog_taxonomy_iam_member" "members" { taxonomy = google_data_catalog_taxonomy.default.id role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } resource "google_data_catalog_policy_tag_iam_binding" "authoritative" { diff --git a/modules/data-catalog-policy-tag/variables.tf b/modules/data-catalog-policy-tag/variables.tf index c7c8eb07..2b799a96 100644 --- a/modules/data-catalog-policy-tag/variables.tf +++ b/modules/data-catalog-policy-tag/variables.tf @@ -55,6 +55,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/dataplex-datascan/README.md b/modules/dataplex-datascan/README.md index 65436af6..ca8afca7 100644 --- a/modules/dataplex-datascan/README.md +++ b/modules/dataplex-datascan/README.md @@ -433,9 +433,9 @@ module "dataplex-datascan" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [data](variables.tf#L17) | The data source for DataScan. The source can be either a Dataplex `entity` or a BigQuery `resource`. | object({…}) | ✓ | | -| [name](variables.tf#L156) | Name of Dataplex Scan. | string | ✓ | | -| [project_id](variables.tf#L167) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | -| [region](variables.tf#L172) | Region for the Dataplex DataScan. | string | ✓ | | +| [name](variables.tf#L161) | Name of Dataplex Scan. | string | ✓ | | +| [project_id](variables.tf#L172) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | +| [region](variables.tf#L177) | Region for the Dataplex DataScan. | string | ✓ | | | [data_profile_spec](variables.tf#L29) | DataProfileScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec. | object({…}) | | null | | [data_quality_spec](variables.tf#L38) | DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | | [data_quality_spec_file](variables.tf#L80) | Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | @@ -445,11 +445,11 @@ module "dataplex-datascan" { | [iam](variables.tf#L107) | Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L114) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L121) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L127) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_policy](variables.tf#L137) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [incremental_field](variables.tf#L143) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | -| [labels](variables.tf#L149) | Resource labels. | map(string) | | {} | -| [prefix](variables.tf#L161) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | +| [iam_members](variables.tf#L127) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_policy](variables.tf#L142) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | +| [incremental_field](variables.tf#L148) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | +| [labels](variables.tf#L154) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L166) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | ## Outputs diff --git a/modules/dataplex-datascan/iam.tf b/modules/dataplex-datascan/iam.tf index e1a7c057..2a6c4c78 100644 --- a/modules/dataplex-datascan/iam.tf +++ b/modules/dataplex-datascan/iam.tf @@ -76,6 +76,14 @@ resource "google_dataplex_datascan_iam_member" "members" { data_scan_id = google_dataplex_datascan.datascan.data_scan_id role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } resource "google_dataplex_datascan_iam_policy" "authoritative_for_resource" { diff --git a/modules/dataplex-datascan/variables.tf b/modules/dataplex-datascan/variables.tf index 47ca7332..f40ca4ec 100644 --- a/modules/dataplex-datascan/variables.tf +++ b/modules/dataplex-datascan/variables.tf @@ -129,6 +129,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 3e454e50..b9dfe3d0 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -165,17 +165,17 @@ module "processing-dp-cluster" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L222) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L237) | Project ID. | string | ✓ | | -| [region](variables.tf#L242) | Dataproc region. | string | ✓ | | +| [name](variables.tf#L227) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L242) | Project ID. | string | ✓ | | +| [region](variables.tf#L247) | Dataproc region. | string | ✓ | | | [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | | [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L199) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_members](variables.tf#L206) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [labels](variables.tf#L216) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | -| [prefix](variables.tf#L227) | Optional prefix used to generate project id and name. | string | | null | -| [service_account](variables.tf#L247) | Service account to set on the Dataproc cluster. | string | | null | +| [iam_members](variables.tf#L206) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [labels](variables.tf#L221) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [prefix](variables.tf#L232) | Optional prefix used to generate project id and name. | string | | null | +| [service_account](variables.tf#L252) | Service account to set on the Dataproc cluster. | string | | null | ## Outputs diff --git a/modules/dataproc/iam.tf b/modules/dataproc/iam.tf index 84d49f35..04756192 100644 --- a/modules/dataproc/iam.tf +++ b/modules/dataproc/iam.tf @@ -72,4 +72,12 @@ resource "google_dataproc_cluster_iam_member" "members" { cluster = google_dataproc_cluster.cluster.name role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index b28b9c27..37aad39f 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -208,6 +208,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/folder/README.md b/modules/folder/README.md index b46c5e8b..03c2a755 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -325,17 +325,17 @@ module "folder" { | [iam](variables.tf#L44) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L51) | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L58) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L65) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_policy](variables.tf#L75) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [id](variables.tf#L81) | Folder ID in case you use folder_create=false. | string | | null | -| [logging_data_access](variables.tf#L87) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L102) | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L109) | Logging sinks to create for the organization. | map(object({…})) | | {} | -| [name](variables.tf#L139) | Folder name. | string | | null | -| [org_policies](variables.tf#L145) | Organization policies applied to this folder keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L172) | Path containing org policies in YAML format. | string | | null | -| [parent](variables.tf#L178) | Parent in folders/folder_id or organizations/org_id format. | string | | null | -| [tag_bindings](variables.tf#L188) | Tag bindings for this folder, in key => tag value id format. | map(string) | | null | +| [iam_members](variables.tf#L65) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_policy](variables.tf#L80) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | +| [id](variables.tf#L86) | Folder ID in case you use folder_create=false. | string | | null | +| [logging_data_access](variables.tf#L92) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L107) | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L114) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [name](variables.tf#L144) | Folder name. | string | | null | +| [org_policies](variables.tf#L150) | Organization policies applied to this folder keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L177) | Path containing org policies in YAML format. | string | | null | +| [parent](variables.tf#L183) | Parent in folders/folder_id or organizations/org_id format. | string | | null | +| [tag_bindings](variables.tf#L193) | Tag bindings for this folder, in key => tag value id format. | map(string) | | null | ## Outputs diff --git a/modules/folder/iam.tf b/modules/folder/iam.tf index 6b8fc1b1..cc457e28 100644 --- a/modules/folder/iam.tf +++ b/modules/folder/iam.tf @@ -69,6 +69,14 @@ resource "google_folder_iam_member" "members" { folder = local.folder.name role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } resource "google_folder_iam_policy" "authoritative" { diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf index be7aad32..df057995 100644 --- a/modules/folder/variables.tf +++ b/modules/folder/variables.tf @@ -67,6 +67,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/iam-service-account/README.md b/modules/iam-service-account/README.md index 061b651e..6c03b824 100644 --- a/modules/iam-service-account/README.md +++ b/modules/iam-service-account/README.md @@ -41,8 +41,8 @@ module "myproject-default-service-accounts" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L101) | Name of the service account to create. | string | ✓ | | -| [project_id](variables.tf#L116) | Project id where service account will be created. | string | ✓ | | +| [name](variables.tf#L106) | Name of the service account to create. | string | ✓ | | +| [project_id](variables.tf#L121) | Project id where service account will be created. | string | ✓ | | | [description](variables.tf#L17) | Optional description. | string | | null | | [display_name](variables.tf#L23) | Display name of the service account to create. | string | | "Terraform-managed." | | [generate_key](variables.tf#L29) | Generate a key for service account. | bool | | false | @@ -50,14 +50,14 @@ module "myproject-default-service-accounts" { | [iam_additive](variables.tf#L42) | IAM additive bindings on the service account in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_billing_roles](variables.tf#L49) | Billing account roles granted to this service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | | [iam_folder_roles](variables.tf#L56) | Folder roles granted to this service account, by folder id. Non-authoritative. | map(list(string)) | | {} | -| [iam_members](variables.tf#L63) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_organization_roles](variables.tf#L73) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | -| [iam_project_roles](variables.tf#L80) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | -| [iam_sa_roles](variables.tf#L87) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | -| [iam_storage_roles](variables.tf#L94) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | -| [prefix](variables.tf#L106) | Prefix applied to service account names. | string | | null | -| [public_keys_directory](variables.tf#L121) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | -| [service_account_create](variables.tf#L127) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | +| [iam_members](variables.tf#L63) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_organization_roles](variables.tf#L78) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | +| [iam_project_roles](variables.tf#L85) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | +| [iam_sa_roles](variables.tf#L92) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | +| [iam_storage_roles](variables.tf#L99) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | +| [prefix](variables.tf#L111) | Prefix applied to service account names. | string | | null | +| [public_keys_directory](variables.tf#L126) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | +| [service_account_create](variables.tf#L132) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | ## Outputs diff --git a/modules/iam-service-account/iam.tf b/modules/iam-service-account/iam.tf index ae388c8e..a92d5e2a 100644 --- a/modules/iam-service-account/iam.tf +++ b/modules/iam-service-account/iam.tf @@ -139,6 +139,14 @@ resource "google_service_account_iam_member" "members" { service_account_id = each.value.entity role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } resource "google_storage_bucket_iam_member" "bucket-roles" { diff --git a/modules/iam-service-account/variables.tf b/modules/iam-service-account/variables.tf index 3594fbbc..87c350b6 100644 --- a/modules/iam-service-account/variables.tf +++ b/modules/iam-service-account/variables.tf @@ -65,6 +65,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/kms/README.md b/modules/kms/README.md index 7b8eb487..5bd82d5f 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -101,19 +101,19 @@ module "kms" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [keyring](variables.tf#L91) | Keyring attributes. | object({…}) | ✓ | | -| [project_id](variables.tf#L114) | Project id where the keyring will be created. | string | ✓ | | +| [keyring](variables.tf#L101) | Keyring attributes. | object({…}) | ✓ | | +| [project_id](variables.tf#L124) | Project id where the keyring will be created. | string | ✓ | | | [iam](variables.tf#L17) | Keyring IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L23) | Keyring IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_members](variables.tf#L29) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [key_iam](variables.tf#L39) | Key IAM bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [key_iam_additive](variables.tf#L45) | Key IAM additive bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [key_iam_members](variables.tf#L51) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [key_purpose](variables.tf#L62) | Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | map(object({…})) | | {} | -| [key_purpose_defaults](variables.tf#L74) | Defaults used for key purpose when not defined at the key level. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | object({…}) | | {…} | -| [keyring_create](variables.tf#L99) | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | -| [keys](variables.tf#L105) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | -| [tag_bindings](variables.tf#L119) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | null | +| [iam_members](variables.tf#L29) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [key_iam](variables.tf#L44) | Key IAM bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [key_iam_additive](variables.tf#L50) | Key IAM additive bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [key_iam_members](variables.tf#L56) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [key_purpose](variables.tf#L72) | Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | map(object({…})) | | {} | +| [key_purpose_defaults](variables.tf#L84) | Defaults used for key purpose when not defined at the key level. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | object({…}) | | {…} | +| [keyring_create](variables.tf#L109) | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | +| [keys](variables.tf#L115) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L129) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | null | ## Outputs diff --git a/modules/kms/iam.tf b/modules/kms/iam.tf index 3da20d08..9289a4e8 100644 --- a/modules/kms/iam.tf +++ b/modules/kms/iam.tf @@ -67,6 +67,14 @@ resource "google_kms_key_ring_iam_member" "members" { key_ring_id = local.keyring.id role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } resource "google_kms_crypto_key_iam_binding" "default" { @@ -94,4 +102,12 @@ resource "google_kms_crypto_key_iam_member" "members" { crypto_key_id = google_kms_crypto_key.default[each.value.key].id role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } diff --git a/modules/kms/variables.tf b/modules/kms/variables.tf index 01e96c04..c34aaaac 100644 --- a/modules/kms/variables.tf +++ b/modules/kms/variables.tf @@ -31,6 +31,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} @@ -54,6 +59,11 @@ variable "key_iam_members" { key = string member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 8106e1d9..0ccfeb86 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -541,11 +541,11 @@ module "vpc" { | [shared_vpc_service_projects](variables.tf#L161) | Shared VPC service projects to register with this host. | list(string) | | [] | | [subnet_iam](variables.tf#L167) | Subnet IAM bindings in {REGION/NAME => {ROLE => [MEMBERS]} format. | map(map(list(string))) | | {} | | [subnet_iam_additive](variables.tf#L173) | Subnet IAM additive bindings in {REGION/NAME => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [subnet_iam_members](variables.tf#L180) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [subnets](variables.tf#L191) | Subnet configuration. | list(object({…})) | | [] | -| [subnets_proxy_only](variables.tf#L217) | List of proxy-only subnets for Regional HTTPS or Internal HTTPS load balancers. Note: Only one proxy-only subnet for each VPC network in each region can be active. | list(object({…})) | | [] | -| [subnets_psc](variables.tf#L229) | List of subnets for Private Service Connect service producers. | list(object({…})) | | [] | -| [vpc_create](variables.tf#L240) | Create VPC. When set to false, uses a data source to reference existing VPC. | bool | | true | +| [subnet_iam_members](variables.tf#L180) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [subnets](variables.tf#L196) | Subnet configuration. | list(object({…})) | | [] | +| [subnets_proxy_only](variables.tf#L222) | List of proxy-only subnets for Regional HTTPS or Internal HTTPS load balancers. Note: Only one proxy-only subnet for each VPC network in each region can be active. | list(object({…})) | | [] | +| [subnets_psc](variables.tf#L234) | List of subnets for Private Service Connect service producers. | list(object({…})) | | [] | +| [vpc_create](variables.tf#L245) | Create VPC. When set to false, uses a data source to reference existing VPC. | bool | | true | ## Outputs diff --git a/modules/net-vpc/subnets.tf b/modules/net-vpc/subnets.tf index 16f6398c..b6eab182 100644 --- a/modules/net-vpc/subnets.tf +++ b/modules/net-vpc/subnets.tf @@ -197,4 +197,12 @@ resource "google_compute_subnetwork_iam_member" "members" { region = google_compute_subnetwork.subnetwork[each.value.subnet].region role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index 4e114dfe..c49455fa 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -183,6 +183,11 @@ variable "subnet_iam_members" { member = string role = string subnet = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/organization/README.md b/modules/organization/README.md index d232dfd4..4984d2a8 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -475,7 +475,7 @@ module "org" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [organization_id](variables.tf#L209) | Organization id in organizations/nnnnnn format. | string | ✓ | | +| [organization_id](variables.tf#L214) | Organization id in organizations/nnnnnn format. | string | ✓ | | | [contacts](variables.tf#L17) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | | [custom_roles](variables.tf#L24) | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | | [firewall_policy_associations](variables.tf#L31) | Hierarchical firewall policies to associate to this folder, in association name => policy id format. | map(string) | | {} | @@ -483,18 +483,18 @@ module "org" { | [iam](variables.tf#L45) | IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L52) | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L59) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L66) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_policy](variables.tf#L76) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [logging_data_access](variables.tf#L82) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L97) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L104) | Logging sinks to create for the organization. | map(object({…})) | | {} | -| [network_tags](variables.tf#L134) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | -| [org_policies](variables.tf#L156) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L183) | Path containing org policies in YAML format. | string | | null | -| [org_policy_custom_constraints](variables.tf#L189) | Organization policy custom constraints keyed by constraint name. | map(object({…})) | | {} | -| [org_policy_custom_constraints_data_path](variables.tf#L203) | Path containing org policy custom constraints in YAML format. | string | | null | -| [tag_bindings](variables.tf#L218) | Tag bindings for this organization, in key => tag value id format. | map(string) | | null | -| [tags](variables.tf#L224) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | +| [iam_members](variables.tf#L66) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_policy](variables.tf#L81) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | +| [logging_data_access](variables.tf#L87) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L102) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L109) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [network_tags](variables.tf#L139) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | +| [org_policies](variables.tf#L161) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L188) | Path containing org policies in YAML format. | string | | null | +| [org_policy_custom_constraints](variables.tf#L194) | Organization policy custom constraints keyed by constraint name. | map(object({…})) | | {} | +| [org_policy_custom_constraints_data_path](variables.tf#L208) | Path containing org policy custom constraints in YAML format. | string | | null | +| [tag_bindings](variables.tf#L223) | Tag bindings for this organization, in key => tag value id format. | map(string) | | null | +| [tags](variables.tf#L229) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | ## Outputs diff --git a/modules/organization/iam.tf b/modules/organization/iam.tf index f5d3b2d7..e5ba3771 100644 --- a/modules/organization/iam.tf +++ b/modules/organization/iam.tf @@ -78,6 +78,14 @@ resource "google_organization_iam_member" "members" { org_id = local.organization_id_numeric role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } resource "google_organization_iam_policy" "authoritative" { diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf index 65f0672b..2426f8e9 100644 --- a/modules/organization/variables.tf +++ b/modules/organization/variables.tf @@ -68,6 +68,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/project/README.md b/modules/project/README.md index 8a867b51..8ec483fb 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -154,7 +154,7 @@ module "project" { #### Additive IAM by Binding -When the above approaches to additive IAM are unworkable due to dynamically generated principals, the `iam_members` variable allows specifying individual role/principal pairs using arbitrary keys: +When the above approaches to additive IAM are unworkable due to dynamically generated principals, the `iam_members` variable allows specifying individual role/principal pairs using arbitrary keys. This IAM variable also supports conditions. ```hcl module "project" { @@ -173,10 +173,23 @@ module "project" { member = "user:two@example.org" role = "roles/compute.admin" } + one-delegated-grant = { + member = "user:one@example.org" + role = "roles/resourcemanager.projectIamAdmin" + condition = { + title = "delegated_network_user_one" + expression = <<-END + api.getAttribute( + 'iam.googleapis.com/modifiedGrantsByRole', [] + ).hasOnly([ + 'roles/compute.networkAdmin' + ]) + END + } + } } - } -# tftest modules=1 resources=4 inventory=iam-members.yaml +# tftest modules=1 resources=5 inventory=iam-members.yaml ``` ### Service Identities and Authoritative IAM @@ -658,7 +671,7 @@ output "compute_robot" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L171) | Project name and id suffix. | string | ✓ | | +| [name](variables.tf#L176) | Project name and id suffix. | string | ✓ | | | [auto_create_network](variables.tf#L17) | Whether to create the default network for the project. | bool | | false | | [billing_account](variables.tf#L23) | Billing account id. | string | | null | | [contacts](variables.tf#L29) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | @@ -669,31 +682,31 @@ output "compute_robot" { | [iam](variables.tf#L62) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L69) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L76) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L82) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_policy](variables.tf#L92) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [labels](variables.tf#L98) | Resource labels. | map(string) | | {} | -| [lien_reason](variables.tf#L105) | If non-empty, creates a project lien with this description. | string | | "" | -| [logging_data_access](variables.tf#L111) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L126) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L133) | Logging sinks to create for this project. | map(object({…})) | | {} | -| [metric_scopes](variables.tf#L164) | List of projects that will act as metric scopes for this project. | list(string) | | [] | -| [org_policies](variables.tf#L176) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L203) | Path containing org policies in YAML format. | string | | null | -| [oslogin](variables.tf#L209) | Enable OS Login. | bool | | false | -| [oslogin_admins](variables.tf#L215) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | -| [oslogin_users](variables.tf#L223) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | -| [parent](variables.tf#L230) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | -| [prefix](variables.tf#L240) | Optional prefix used to generate project id and name. | string | | null | -| [project_create](variables.tf#L250) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | -| [service_config](variables.tf#L256) | Configure service API activation. | object({…}) | | {…} | -| [service_encryption_key_ids](variables.tf#L268) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | -| [service_perimeter_bridges](variables.tf#L275) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | -| [service_perimeter_standard](variables.tf#L282) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | -| [services](variables.tf#L288) | Service APIs to enable. | list(string) | | [] | -| [shared_vpc_host_config](variables.tf#L294) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | -| [shared_vpc_service_config](variables.tf#L303) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | -| [skip_delete](variables.tf#L325) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | -| [tag_bindings](variables.tf#L331) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | +| [iam_members](variables.tf#L82) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [iam_policy](variables.tf#L97) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | +| [labels](variables.tf#L103) | Resource labels. | map(string) | | {} | +| [lien_reason](variables.tf#L110) | If non-empty, creates a project lien with this description. | string | | "" | +| [logging_data_access](variables.tf#L116) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L131) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L138) | Logging sinks to create for this project. | map(object({…})) | | {} | +| [metric_scopes](variables.tf#L169) | List of projects that will act as metric scopes for this project. | list(string) | | [] | +| [org_policies](variables.tf#L181) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L208) | Path containing org policies in YAML format. | string | | null | +| [oslogin](variables.tf#L214) | Enable OS Login. | bool | | false | +| [oslogin_admins](variables.tf#L220) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | +| [oslogin_users](variables.tf#L228) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | +| [parent](variables.tf#L235) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | +| [prefix](variables.tf#L245) | Optional prefix used to generate project id and name. | string | | null | +| [project_create](variables.tf#L255) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | +| [service_config](variables.tf#L261) | Configure service API activation. | object({…}) | | {…} | +| [service_encryption_key_ids](variables.tf#L273) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | +| [service_perimeter_bridges](variables.tf#L280) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | +| [service_perimeter_standard](variables.tf#L287) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | +| [services](variables.tf#L293) | Service APIs to enable. | list(string) | | [] | +| [shared_vpc_host_config](variables.tf#L299) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | +| [shared_vpc_service_config](variables.tf#L308) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | +| [skip_delete](variables.tf#L330) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | +| [tag_bindings](variables.tf#L336) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | ## Outputs diff --git a/modules/project/iam.tf b/modules/project/iam.tf index 7b67d895..0918b552 100644 --- a/modules/project/iam.tf +++ b/modules/project/iam.tf @@ -102,6 +102,14 @@ resource "google_project_iam_member" "members" { project = local.project.project_id role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } depends_on = [ google_project_service.project_services, google_project_iam_custom_role.roles diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 4ab51062..18f55886 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -84,6 +84,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/modules/source-repository/README.md b/modules/source-repository/README.md index 4cfae8ea..01fff2d8 100644 --- a/modules/source-repository/README.md +++ b/modules/source-repository/README.md @@ -75,14 +75,14 @@ module "repo" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L54) | Repository name. | string | ✓ | | -| [project_id](variables.tf#L59) | Project used for resources. | string | ✓ | | +| [name](variables.tf#L59) | Repository name. | string | ✓ | | +| [project_id](variables.tf#L64) | Project used for resources. | string | ✓ | | | [group_iam](variables.tf#L17) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L24) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L31) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive_members](variables.tf#L38) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L44) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [triggers](variables.tf#L64) | Cloud Build triggers. | map(object({…})) | | {} | +| [iam_members](variables.tf#L44) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | +| [triggers](variables.tf#L69) | Cloud Build triggers. | map(object({…})) | | {} | ## Outputs diff --git a/modules/source-repository/iam.tf b/modules/source-repository/iam.tf index f1ec3ebc..75d00fd1 100644 --- a/modules/source-repository/iam.tf +++ b/modules/source-repository/iam.tf @@ -72,4 +72,12 @@ resource "google_sourcerepo_repository_iam_member" "members" { repository = google_sourcerepo_repository.default.name role = each.value.role member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } diff --git a/modules/source-repository/variables.tf b/modules/source-repository/variables.tf index 74e44aa4..61975fb9 100644 --- a/modules/source-repository/variables.tf +++ b/modules/source-repository/variables.tf @@ -46,6 +46,11 @@ variable "iam_members" { type = map(object({ member = string role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) })) nullable = false default = {} diff --git a/tests/modules/project/examples/iam-members.yaml b/tests/modules/project/examples/iam-members.yaml index b80fc2bf..41f48a56 100644 --- a/tests/modules/project/examples/iam-members.yaml +++ b/tests/modules/project/examples/iam-members.yaml @@ -23,6 +23,15 @@ values: project_id: project-example skip_delete: false timeouts: null + module.project.google_project_iam_member.members["one-delegated-grant"]: + condition: + - description: null + expression: "api.getAttribute(\n 'iam.googleapis.com/modifiedGrantsByRole',\ + \ []\n).hasOnly([\n 'roles/compute.networkAdmin'\n])\n" + title: delegated_network_user_one + member: user:one@example.org + project: project-example + role: roles/resourcemanager.projectIamAdmin module.project.google_project_iam_member.members["one-owner"]: condition: [] member: user:one@example.org @@ -41,8 +50,9 @@ values: counts: google_project: 1 - google_project_iam_member: 3 + google_project_iam_member: 4 modules: 1 - resources: 4 + resources: 5 outputs: {} + From a7fd3e26162283496112e02bf6f6175a17d30b82 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 09:25:44 +0200 Subject: [PATCH 12/46] fixed file name --- modules/gcve-private-cloud/{output.tf => outputs.tf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename modules/gcve-private-cloud/{output.tf => outputs.tf} (100%) diff --git a/modules/gcve-private-cloud/output.tf b/modules/gcve-private-cloud/outputs.tf similarity index 100% rename from modules/gcve-private-cloud/output.tf rename to modules/gcve-private-cloud/outputs.tf From 77d80a40c389e57e12759d3cfe68ae4756bedd64 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 09:34:18 +0200 Subject: [PATCH 13/46] added link to GCVE module --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c7b6d94c..6d67d06b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The current list of modules supports most of the core foundational and networkin Currently available modules: - **foundational** - [billing budget](./modules/billing-budget), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source) -- **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [Internal Application LB](./modules/net-lb-app-int), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC firewall policy](./modules/net-vpc-firewall-policy), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) +- **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) - **data** - [AlloyDB instance](./modules/alloydb-instance), [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub) - **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository) From e927bf3858a1ec9ff2083f294811ff28d93fca97 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 09:37:48 +0200 Subject: [PATCH 14/46] Fixed typo --- modules/gcve-private-cloud/README.md | 42 ++++++++++++++----------- modules/gcve-private-cloud/main.tf | 15 ++++----- modules/gcve-private-cloud/variables.tf | 10 ++++-- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index 8baad718..ab374c08 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -1,8 +1,8 @@ # Google Cloud VMWare Engine Private Cloud Module -This module implements the creation and management of a Google Cloud VMWare Engine Private Cloud with its management cluster. If configured, it also creates the vmware engine network or it can work with an existing one. The creation of the private connection with the user VPC requires the execution of the [Google SDK command](https://cloud.google.com/sdk/gcloud/reference/vmware/private-connections/create#--routing-mode) the module provides as an output. +This module implements the creation and management of a Google Cloud VMWare Engine Private Cloud with its management cluster. If configured, it also creates the VMWare engine network or it can work with an existing one. The creation of the private connection with the user VPC requires the execution of the [Google SDK command](https://cloud.google.com/sdk/gcloud/reference/vmware/private-connections/create#--routing-mode) the module provides as an output. -Be aware that the deployment of this module might requires up to 2 hours depending on the selected private cloud target zone. +Be aware that the deployment of this module might require up to 2 hours depending on the selected private cloud target zone. ## TOC @@ -11,14 +11,14 @@ Be aware that the deployment of this module might requires up to 2 hours dependi - [Limitations](#limitations) - [Basic Private Cloud Creation](#basic-private-cloud-creation) - [Private Cloud Creation with custom nodes and cores count](#private-cloud-creation-with-custom-nodes-and-cores-count) -- [Files](#files) - [Variables](#variables) +- [Outputs](#outputs) ## Limitations At the moment this module doesn't support the following use cases: -- `Single node private cloud` -- `Stretched private cloud` +- Single node private cloud +- Stretched private cloud ## Basic Private Cloud Creation @@ -70,18 +70,7 @@ module "gcve-pc" { } # tftest modules=1 resources=2 inventory=custom.yaml ``` - - -## Files - -| name | description | resources | -|---|---|---| -| [main.tf](./main.tf) | Module-level locals and resources. | google_vmwareengine_network · google_vmwareengine_private_cloud | -| [output.tf](./output.tf) | None | | -| [variables.tf](./variables.tf) | Module variables. | | -| [versions.tf](./versions.tf) | Version pins. | | - ## Variables | name | description | type | required | default | @@ -89,9 +78,24 @@ module "gcve-pc" { | [management_cidr](variables.tf#L23) | vSphere/vSAN subnets CIDR range. | string | ✓ | | | [name](variables.tf#L42) | Private cloud name. | string | ✓ | | | [project_id](variables.tf#L74) | Project id. | string | ✓ | | -| [zone](variables.tf#L85) | Private cloud zone. | string | ✓ | | +| [zone](variables.tf#L91) | Private cloud zone. | string | ✓ | | | [description](variables.tf#L17) | Private cloud description. | string | | "Terraform-managed." | | [management_cluster_config](variables.tf#L28) | Management cluster configuration. | object({…}) | | {…} | -| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | -| [vmwareengine_network_create](variables.tf#L79) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | +| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [vmw_network_create](variables.tf#L79) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | +| [vmw_network_description](variables.tf#L85) | VMware Engine network description. | string | | "Terraform-managed." | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [hcx](outputs.tf#L17) | Details about a HCX Cloud Manager appliance. | | +| [id](outputs.tf#L22) | ID of the private cloud | | +| [management_cluster](outputs.tf#L27) | Details of the management cluster of the private cloud | | +| [network_config](outputs.tf#L32) | Details about the network configuration of the private cloud | | +| [nsx](outputs.tf#L37) | Details about a NSX Manager appliance. | | +| [private-cloud](outputs.tf#L42) | The private cloud resource | | +| [private_connections_setup](outputs.tf#L57) | Cloud SDK commands for the private connections manual setup. | | +| [state](outputs.tf#L52) | Details about the state of the private cloud | | +| [vcenter](outputs.tf#L47) | Details about a vCenter Server management appliance. | | diff --git a/modules/gcve-private-cloud/main.tf b/modules/gcve-private-cloud/main.tf index 3f184475..2568f29d 100644 --- a/modules/gcve-private-cloud/main.tf +++ b/modules/gcve-private-cloud/main.tf @@ -16,18 +16,19 @@ locals { region = join("-", slice(split("-", "${var.zone}"), 0, 2)) - vmwareengine_network = ( - var.vmwareengine_network_create + vmw_network = ( + var.vmw_network_create ? try(google_vmwareengine_network.private-cloud-network.0, null) : try(data.google_vmwareengine_network.private-cloud-network.0, null) ) psa_peering = { - for k, v in data.google_compute_network_peering.psa_peering : k => slice(split("/", "${v.peer_network}"), 6, 7)[0] + for k, v in data.google_compute_network_peering.psa_peering : + k => slice(split("/", "${v.peer_network}"), 6, 7)[0] } } data "google_vmwareengine_network" "private-cloud-network" { - count = var.vmwareengine_network_create ? 0 : 1 + count = var.vmw_network_create ? 0 : 1 provider = google-beta project = var.project_id name = "${local.region}-default" @@ -50,7 +51,7 @@ resource "google_vmwareengine_private_cloud" "private-cloud" { network_config { management_cidr = var.management_cidr - vmware_engine_network = local.vmwareengine_network.id + vmware_engine_network = local.vmw_network.id } management_cluster { @@ -64,11 +65,11 @@ resource "google_vmwareengine_private_cloud" "private-cloud" { } resource "google_vmwareengine_network" "private-cloud-network" { - count = var.vmwareengine_network_create ? 1 : 0 + count = var.vmw_network_create ? 1 : 0 provider = google-beta project = var.project_id name = "${local.region}-default" location = local.region type = "LEGACY" - description = "Private cloud ${var.name} network." + description = var.vmw_network_description } diff --git a/modules/gcve-private-cloud/variables.tf b/modules/gcve-private-cloud/variables.tf index 05183e0d..61e5b1d4 100644 --- a/modules/gcve-private-cloud/variables.tf +++ b/modules/gcve-private-cloud/variables.tf @@ -48,9 +48,9 @@ variable "private_connections" { description = "VMWare private connections configuration. It is used to create the gcloud command printed as output." type = map(object({ name = string - description = optional(string, "Terraform-managed.") network_self_link = string peering = string + description = optional(string, "Terraform-managed.") type = optional(string, "REGIONAL") routing_mode = optional(string, "PRIVATE_SERVICE_ACCESS") })) @@ -76,12 +76,18 @@ variable "project_id" { type = string } -variable "vmwareengine_network_create" { +variable "vmw_network_create" { description = "Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network." type = bool default = true } +variable "vmw_network_description" { + description = " VMware Engine network description." + type = string + default = "Terraform-managed." +} + variable "zone" { description = "Private cloud zone." type = string From dcb3c327616158ec7b295f18c7134e32b43a4a66 Mon Sep 17 00:00:00 2001 From: Stefan Moser <6841360+sm3142@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:03:23 +0200 Subject: [PATCH 15/46] fix null object exception in bootstrap output when using cloudsource repos (#1597) --- fast/stages/0-bootstrap/README.md | 20 ++++++++++---------- fast/stages/0-bootstrap/outputs.tf | 4 +++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/fast/stages/0-bootstrap/README.md b/fast/stages/0-bootstrap/README.md index a9a28b1b..ffaeb17d 100644 --- a/fast/stages/0-bootstrap/README.md +++ b/fast/stages/0-bootstrap/README.md @@ -529,14 +529,14 @@ The remaining configuration is manual, as it regards the repositories themselves | name | description | sensitive | consumers | |---|---|:---:|---| -| [automation](outputs.tf#L100) | Automation resources. | | | -| [billing_dataset](outputs.tf#L105) | BigQuery dataset prepared for billing export. | | | -| [cicd_repositories](outputs.tf#L110) | CI/CD repository configurations. | | | -| [custom_roles](outputs.tf#L122) | Organization-level custom roles. | | | -| [federated_identity](outputs.tf#L127) | Workload Identity Federation pool and providers. | | | -| [outputs_bucket](outputs.tf#L137) | GCS bucket where generated output files are stored. | | | -| [project_ids](outputs.tf#L142) | Projects created by this stage. | | | -| [providers](outputs.tf#L152) | Terraform provider files for this stage and dependent stages. | ✓ | stage-01 | -| [service_accounts](outputs.tf#L159) | Automation service accounts created by this stage. | | | -| [tfvars](outputs.tf#L168) | Terraform variable files for the following stages. | ✓ | | +| [automation](outputs.tf#L102) | Automation resources. | | | +| [billing_dataset](outputs.tf#L107) | BigQuery dataset prepared for billing export. | | | +| [cicd_repositories](outputs.tf#L112) | CI/CD repository configurations. | | | +| [custom_roles](outputs.tf#L124) | Organization-level custom roles. | | | +| [federated_identity](outputs.tf#L129) | Workload Identity Federation pool and providers. | | | +| [outputs_bucket](outputs.tf#L139) | GCS bucket where generated output files are stored. | | | +| [project_ids](outputs.tf#L144) | Projects created by this stage. | | | +| [providers](outputs.tf#L154) | Terraform provider files for this stage and dependent stages. | ✓ | stage-01 | +| [service_accounts](outputs.tf#L161) | Automation service accounts created by this stage. | | | +| [tfvars](outputs.tf#L170) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages/0-bootstrap/outputs.tf b/fast/stages/0-bootstrap/outputs.tf index ea980fca..70d84449 100644 --- a/fast/stages/0-bootstrap/outputs.tf +++ b/fast/stages/0-bootstrap/outputs.tf @@ -22,7 +22,9 @@ locals { "${path.module}/templates/workflow-${v.type}.yaml", { # If users give a list of custom audiences we set by default the first element. # If no audiences are given, we set https://iam.googleapis.com/{PROVIDER_NAME} - audiences = local.cicd_providers[v["identity_provider"]].audiences + audiences = try( + local.cicd_providers[v["identity_provider"]].audiences, "" + ) identity_provider = try( local.cicd_providers[v["identity_provider"]].name, "" ) From 8e9d544be675c957285f4116de9afa97f2d74b95 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 12:04:48 +0200 Subject: [PATCH 16/46] fixed description --- tests/modules/gcve_private_cloud/examples/basic.yaml | 2 ++ tests/modules/gcve_private_cloud/examples/custom.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/modules/gcve_private_cloud/examples/basic.yaml b/tests/modules/gcve_private_cloud/examples/basic.yaml index c4d9e3d5..b91dcf58 100644 --- a/tests/modules/gcve_private_cloud/examples/basic.yaml +++ b/tests/modules/gcve_private_cloud/examples/basic.yaml @@ -14,12 +14,14 @@ values: module.gcve-pc.google_vmwareengine_network.private-cloud-network[0]: + description: Terraform-managed. location: asia-southeast1 name: asia-southeast1-default project: gcve-test-project timeouts: null type: LEGACY module.gcve-pc.google_vmwareengine_private_cloud.private-cloud: + description: Terraform-managed. location: asia-southeast1-a management_cluster: - cluster_id: gcve-pc-mgmt-cluster diff --git a/tests/modules/gcve_private_cloud/examples/custom.yaml b/tests/modules/gcve_private_cloud/examples/custom.yaml index 60bbf682..4c813e2c 100644 --- a/tests/modules/gcve_private_cloud/examples/custom.yaml +++ b/tests/modules/gcve_private_cloud/examples/custom.yaml @@ -14,7 +14,7 @@ values: module.gcve-pc.google_vmwareengine_network.private-cloud-network[0]: - description: Private cloud gcve-pc network. + description: Terraform-managed. location: asia-southeast1 name: asia-southeast1-default project: gcve-test-project From 0e2ee8bcebe9017862669c68e38114692ac5f7e2 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 12:05:44 +0200 Subject: [PATCH 17/46] fixed typo --- modules/gcve-private-cloud/README.md | 11 +++++------ modules/gcve-private-cloud/main.tf | 2 +- modules/gcve-private-cloud/variables.tf | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index ab374c08..df472334 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -2,12 +2,11 @@ This module implements the creation and management of a Google Cloud VMWare Engine Private Cloud with its management cluster. If configured, it also creates the VMWare engine network or it can work with an existing one. The creation of the private connection with the user VPC requires the execution of the [Google SDK command](https://cloud.google.com/sdk/gcloud/reference/vmware/private-connections/create#--routing-mode) the module provides as an output. +To undersatnd the limits and to propertly configure the vSphere/vSAN subnets CIDR range please refer to the [GCVE public documetation](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). + Be aware that the deployment of this module might require up to 2 hours depending on the selected private cloud target zone. -## TOC - -- [TOC](#toc) - [Limitations](#limitations) - [Basic Private Cloud Creation](#basic-private-cloud-creation) - [Private Cloud Creation with custom nodes and cores count](#private-cloud-creation-with-custom-nodes-and-cores-count) @@ -34,7 +33,7 @@ module "gcve-pc" { transit-conn1 = { name = "transit-conn1", network_self_link = "projects/test-prj-elia-01/global/networks/default", - peering = "servicenetworking-googleapis-com" + peering_name = "servicenetworking-googleapis-com" type = "PRIVATE_SERVICE_ACCESS", routing_mode = "REGIONAL" } @@ -62,7 +61,7 @@ module "gcve-pc" { transit-conn1 = { name = "transit-conn1", network_self_link = "projects/test-prj-elia-01/global/networks/default", - peering = "servicenetworking-googleapis-com" + peering_name = "servicenetworking-googleapis-com" type = "PRIVATE_SERVICE_ACCESS", routing_mode = "REGIONAL" } @@ -81,7 +80,7 @@ module "gcve-pc" { | [zone](variables.tf#L91) | Private cloud zone. | string | ✓ | | | [description](variables.tf#L17) | Private cloud description. | string | | "Terraform-managed." | | [management_cluster_config](variables.tf#L28) | Management cluster configuration. | object({…}) | | {…} | -| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | | [vmw_network_create](variables.tf#L79) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | | [vmw_network_description](variables.tf#L85) | VMware Engine network description. | string | | "Terraform-managed." | diff --git a/modules/gcve-private-cloud/main.tf b/modules/gcve-private-cloud/main.tf index 2568f29d..13b915b7 100644 --- a/modules/gcve-private-cloud/main.tf +++ b/modules/gcve-private-cloud/main.tf @@ -38,7 +38,7 @@ data "google_vmwareengine_network" "private-cloud-network" { data "google_compute_network_peering" "psa_peering" { for_each = var.private_connections - name = each.value.peering + name = each.value.peering_name network = each.value.network_self_link } diff --git a/modules/gcve-private-cloud/variables.tf b/modules/gcve-private-cloud/variables.tf index 61e5b1d4..c1586f41 100644 --- a/modules/gcve-private-cloud/variables.tf +++ b/modules/gcve-private-cloud/variables.tf @@ -49,7 +49,7 @@ variable "private_connections" { type = map(object({ name = string network_self_link = string - peering = string + peering_name = string description = optional(string, "Terraform-managed.") type = optional(string, "REGIONAL") routing_mode = optional(string, "PRIVATE_SERVICE_ACCESS") From 3feaad0c1cadff21552b1ceb0ffaf5ee37b4a102 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 12:10:44 +0200 Subject: [PATCH 18/46] fixed variable name --- modules/gcve-private-cloud/README.md | 4 ++-- modules/gcve-private-cloud/main.tf | 2 +- modules/gcve-private-cloud/variables.tf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index df472334..2b85cd31 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -33,7 +33,7 @@ module "gcve-pc" { transit-conn1 = { name = "transit-conn1", network_self_link = "projects/test-prj-elia-01/global/networks/default", - peering_name = "servicenetworking-googleapis-com" + psa_peering = "servicenetworking-googleapis-com" type = "PRIVATE_SERVICE_ACCESS", routing_mode = "REGIONAL" } @@ -61,7 +61,7 @@ module "gcve-pc" { transit-conn1 = { name = "transit-conn1", network_self_link = "projects/test-prj-elia-01/global/networks/default", - peering_name = "servicenetworking-googleapis-com" + psa_peering = "servicenetworking-googleapis-com" type = "PRIVATE_SERVICE_ACCESS", routing_mode = "REGIONAL" } diff --git a/modules/gcve-private-cloud/main.tf b/modules/gcve-private-cloud/main.tf index 13b915b7..c7837978 100644 --- a/modules/gcve-private-cloud/main.tf +++ b/modules/gcve-private-cloud/main.tf @@ -38,7 +38,7 @@ data "google_vmwareengine_network" "private-cloud-network" { data "google_compute_network_peering" "psa_peering" { for_each = var.private_connections - name = each.value.peering_name + name = each.value.psa_peering network = each.value.network_self_link } diff --git a/modules/gcve-private-cloud/variables.tf b/modules/gcve-private-cloud/variables.tf index c1586f41..69c083a4 100644 --- a/modules/gcve-private-cloud/variables.tf +++ b/modules/gcve-private-cloud/variables.tf @@ -49,7 +49,7 @@ variable "private_connections" { type = map(object({ name = string network_self_link = string - peering_name = string + psa_peering = string description = optional(string, "Terraform-managed.") type = optional(string, "REGIONAL") routing_mode = optional(string, "PRIVATE_SERVICE_ACCESS") From 739bbf1aefcc8e4b0dfd6b79055ed6f539de3c5f Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 12:11:42 +0200 Subject: [PATCH 19/46] fixed variable name --- modules/gcve-private-cloud/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index 2b85cd31..4c9785f3 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -80,7 +80,7 @@ module "gcve-pc" { | [zone](variables.tf#L91) | Private cloud zone. | string | ✓ | | | [description](variables.tf#L17) | Private cloud description. | string | | "Terraform-managed." | | [management_cluster_config](variables.tf#L28) | Management cluster configuration. | object({…}) | | {…} | -| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | | [vmw_network_create](variables.tf#L79) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | | [vmw_network_description](variables.tf#L85) | VMware Engine network description. | string | | "Terraform-managed." | From c6f5d47c66f7e4361f3b197e86f643254bd404eb Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 12:19:47 +0200 Subject: [PATCH 20/46] fixed variable name --- modules/gcve-private-cloud/README.md | 6 +++--- modules/gcve-private-cloud/main.tf | 2 +- modules/gcve-private-cloud/variables.tf | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index 4c9785f3..df472334 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -33,7 +33,7 @@ module "gcve-pc" { transit-conn1 = { name = "transit-conn1", network_self_link = "projects/test-prj-elia-01/global/networks/default", - psa_peering = "servicenetworking-googleapis-com" + peering_name = "servicenetworking-googleapis-com" type = "PRIVATE_SERVICE_ACCESS", routing_mode = "REGIONAL" } @@ -61,7 +61,7 @@ module "gcve-pc" { transit-conn1 = { name = "transit-conn1", network_self_link = "projects/test-prj-elia-01/global/networks/default", - psa_peering = "servicenetworking-googleapis-com" + peering_name = "servicenetworking-googleapis-com" type = "PRIVATE_SERVICE_ACCESS", routing_mode = "REGIONAL" } @@ -80,7 +80,7 @@ module "gcve-pc" { | [zone](variables.tf#L91) | Private cloud zone. | string | ✓ | | | [description](variables.tf#L17) | Private cloud description. | string | | "Terraform-managed." | | [management_cluster_config](variables.tf#L28) | Management cluster configuration. | object({…}) | | {…} | -| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | | [vmw_network_create](variables.tf#L79) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | | [vmw_network_description](variables.tf#L85) | VMware Engine network description. | string | | "Terraform-managed." | diff --git a/modules/gcve-private-cloud/main.tf b/modules/gcve-private-cloud/main.tf index c7837978..13b915b7 100644 --- a/modules/gcve-private-cloud/main.tf +++ b/modules/gcve-private-cloud/main.tf @@ -38,7 +38,7 @@ data "google_vmwareengine_network" "private-cloud-network" { data "google_compute_network_peering" "psa_peering" { for_each = var.private_connections - name = each.value.psa_peering + name = each.value.peering_name network = each.value.network_self_link } diff --git a/modules/gcve-private-cloud/variables.tf b/modules/gcve-private-cloud/variables.tf index 69c083a4..c1586f41 100644 --- a/modules/gcve-private-cloud/variables.tf +++ b/modules/gcve-private-cloud/variables.tf @@ -49,7 +49,7 @@ variable "private_connections" { type = map(object({ name = string network_self_link = string - psa_peering = string + peering_name = string description = optional(string, "Terraform-managed.") type = optional(string, "REGIONAL") routing_mode = optional(string, "PRIVATE_SERVICE_ACCESS") From 6a5739bf91f672cbd2efe75a7195021f346ee2bf Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Thu, 17 Aug 2023 12:25:33 +0200 Subject: [PATCH 21/46] gcve net requirements link --- modules/gcve-private-cloud/README.md | 4 ++-- modules/gcve-private-cloud/variables.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index df472334..0c3fe404 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -2,7 +2,7 @@ This module implements the creation and management of a Google Cloud VMWare Engine Private Cloud with its management cluster. If configured, it also creates the VMWare engine network or it can work with an existing one. The creation of the private connection with the user VPC requires the execution of the [Google SDK command](https://cloud.google.com/sdk/gcloud/reference/vmware/private-connections/create#--routing-mode) the module provides as an output. -To undersatnd the limits and to propertly configure the vSphere/vSAN subnets CIDR range please refer to the [GCVE public documetation](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). +To understand the limits and to propertly configure the vSphere/vSAN subnets CIDR range please refer to the [GCVE public documetation](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). Be aware that the deployment of this module might require up to 2 hours depending on the selected private cloud target zone. @@ -74,7 +74,7 @@ module "gcve-pc" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [management_cidr](variables.tf#L23) | vSphere/vSAN subnets CIDR range. | string | ✓ | | +| [management_cidr](variables.tf#L23) | vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). | string | ✓ | | | [name](variables.tf#L42) | Private cloud name. | string | ✓ | | | [project_id](variables.tf#L74) | Project id. | string | ✓ | | | [zone](variables.tf#L91) | Private cloud zone. | string | ✓ | | diff --git a/modules/gcve-private-cloud/variables.tf b/modules/gcve-private-cloud/variables.tf index c1586f41..945b5b10 100644 --- a/modules/gcve-private-cloud/variables.tf +++ b/modules/gcve-private-cloud/variables.tf @@ -21,7 +21,7 @@ variable "description" { } variable "management_cidr" { - description = "vSphere/vSAN subnets CIDR range." + description = "vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements)." type = string } From 574c7548d8fa098ab46e9724c8271f37ebec351e Mon Sep 17 00:00:00 2001 From: Jay Schwerberg Date: Thu, 17 Aug 2023 14:36:23 -0700 Subject: [PATCH 22/46] feat(cloud-run): add startup cpu boost option --- modules/cloud-run/README.md | 9 +++++---- modules/cloud-run/main.tf | 5 ++++- modules/cloud-run/variables.tf | 6 ++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/modules/cloud-run/README.md b/modules/cloud-run/README.md index 0db9d114..803ddcb6 100644 --- a/modules/cloud-run/README.md +++ b/modules/cloud-run/README.md @@ -352,10 +352,11 @@ module "cloud_run" { | [revision_name](variables.tf#L177) | Revision name. | string | | null | | [service_account](variables.tf#L183) | Service account email. Unused if service account is auto-created. | string | | null | | [service_account_create](variables.tf#L189) | Auto-create service account. | bool | | false | -| [timeout_seconds](variables.tf#L195) | Maximum duration the instance is allowed for responding to a request. | number | | null | -| [traffic](variables.tf#L201) | Traffic steering configuration. If revision name is null the latest revision will be used. | map(object({…})) | | {} | -| [volumes](variables.tf#L212) | Named volumes in containers in name => attributes format. | map(object({…})) | | {} | -| [vpc_connector_create](variables.tf#L226) | Populate this to create a VPC connector. You can then refer to it in the template annotations. | object({…}) | | null | +| [startup_cpu_boost](variables.tf#L195) | Enable startup cpu boost. | bool | | false | +| [timeout_seconds](variables.tf#L201) | Maximum duration the instance is allowed for responding to a request. | number | | null | +| [traffic](variables.tf#L207) | Traffic steering configuration. If revision name is null the latest revision will be used. | map(object({…})) | | {} | +| [volumes](variables.tf#L218) | Named volumes in containers in name => attributes format. | map(object({…})) | | {} | +| [vpc_connector_create](variables.tf#L232) | Populate this to create a VPC connector. You can then refer to it in the template annotations. | object({…}) | | null | ## Outputs diff --git a/modules/cloud-run/main.tf b/modules/cloud-run/main.tf index e5371273..617e67bb 100644 --- a/modules/cloud-run/main.tf +++ b/modules/cloud-run/main.tf @@ -33,7 +33,10 @@ locals { annotations = merge( var.ingress_settings == null ? {} : { "run.googleapis.com/ingress" = var.ingress_settings - } + }, + var.startup_cpu_boost ? { + "run.googleapis.com/startup-cpu-boost" = "true" + } : {} ) _iam_run_invoker_members = concat( lookup(var.iam, "roles/run.invoker", []), diff --git a/modules/cloud-run/variables.tf b/modules/cloud-run/variables.tf index 09d10296..9a4992a7 100644 --- a/modules/cloud-run/variables.tf +++ b/modules/cloud-run/variables.tf @@ -192,6 +192,12 @@ variable "service_account_create" { default = false } +variable "startup_cpu_boost" { + description = "Enable startup cpu boost." + type = bool + default = false +} + variable "timeout_seconds" { description = "Maximum duration the instance is allowed for responding to a request." type = number From ea0de3adbbf36a3d76753f85441b44240a600923 Mon Sep 17 00:00:00 2001 From: Alejandro Leal Date: Fri, 18 Aug 2023 05:51:00 +0000 Subject: [PATCH 23/46] Fixing some typos --- blueprints/data-solutions/data-playground/README.md | 2 +- blueprints/gke/binauthz/README.md | 2 +- fast/stages/2-networking-c-nva/README.md | 2 +- fast/stages/2-networking-e-nva-bgp/README.md | 3 ++- modules/bigtable-instance/README.md | 2 +- modules/projects-data-source/README.md | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/blueprints/data-solutions/data-playground/README.md b/blueprints/data-solutions/data-playground/README.md index c3693ee2..413445d4 100644 --- a/blueprints/data-solutions/data-playground/README.md +++ b/blueprints/data-solutions/data-playground/README.md @@ -24,7 +24,7 @@ If the network_config variable is not provided, one VPC will be created in each ## Deploy your environment -We assume the identiy running the following steps has the following role: +We assume the identity running the following steps has the following role: - resourcemanager.projectCreator in case a new project will be created. - owner on the project in case you use an existing project. diff --git a/blueprints/gke/binauthz/README.md b/blueprints/gke/binauthz/README.md index 740982e6..d4aa5f49 100644 --- a/blueprints/gke/binauthz/README.md +++ b/blueprints/gke/binauthz/README.md @@ -36,7 +36,7 @@ Once the resources have been created, do the following to verify that everything kubectl apply -f tenant-setup.yaml - By applying that manifest thw following is created: + By applying that manifest the following is created: * A namespace called "apis". This is the namespace where the application will be deployed. * A Role and a RoleBinding in previously created namespace so the service account that has been configured for the CD pipeline trigger in Cloud Build is able to deploy the kubernetes application to that namespace. diff --git a/fast/stages/2-networking-c-nva/README.md b/fast/stages/2-networking-c-nva/README.md index a587d68c..778de036 100644 --- a/fast/stages/2-networking-c-nva/README.md +++ b/fast/stages/2-networking-c-nva/README.md @@ -83,7 +83,7 @@ By default, the design assumes the following: - cross-environment traffic and traffic from any untrusted network to any trusted network (and vice versa) pass through the NVAs. For demo purposes, the current NVA performs simple routing/natting only - any traffic from a trusted network to an untrusted network (e.g. Internet) is natted by the NVAs. Users can configure further exclusions -The trusted landing VPC acts as a hub: it bridges internal resources with the outside world and it hosts the shared services consumed by the spoke VPCs, connected to the hub thorugh VPC network peerings. Spokes are used to partition the environments. By default: +The trusted landing VPC acts as a hub: it bridges internal resources with the outside world and it hosts the shared services consumed by the spoke VPCs, connected to the hub through VPC network peerings. Spokes are used to partition the environments. By default: - one spoke VPC hosts the development environment resources - one spoke VPC hosts the production environment resources diff --git a/fast/stages/2-networking-e-nva-bgp/README.md b/fast/stages/2-networking-e-nva-bgp/README.md index a9f444e1..32be44b1 100644 --- a/fast/stages/2-networking-e-nva-bgp/README.md +++ b/fast/stages/2-networking-e-nva-bgp/README.md @@ -37,6 +37,7 @@ The final number of subnets, and their IP addressing will depend on the user-spe - [Design overview and choices](#design-overview-and-choices) - [Multi-regional deployment](#multi-regional-deployment) - [VPC design](#vpc-design) + - [NCC, NVAs and BGP sessions](#ncc-nvas-and-bgp-sessions) - [External connectivity](#external-connectivity) - [Internal connectivity](#internal-connectivity) - [IP ranges, subnetting, routing](#ip-ranges-subnetting-routing) @@ -105,7 +106,7 @@ By default, the design assumes that: - cross-spoke (environment) traffic and traffic from any untrusted network to any trusted network (and vice versa) pass through the NVAs. - any traffic from a trusted network to an untrusted network (e.g. Internet) is natted by the NVAs. Users can configure further exclusions. -The trusted landing VPC acts as a hub: it bridges internal resources with the outside world and it hosts the shared services consumed by the spoke VPCs, connected to the hub thorugh VPC network peerings. Spokes are used to partition the environments. By default: +The trusted landing VPC acts as a hub: it bridges internal resources with the outside world and it hosts the shared services consumed by the spoke VPCs, connected to the hub through VPC network peerings. Spokes are used to partition the environments. By default: - one spoke VPC hosts the development environment resources - one spoke VPC hosts the production environment resources diff --git a/modules/bigtable-instance/README.md b/modules/bigtable-instance/README.md index abc1509f..afe1ec42 100644 --- a/modules/bigtable-instance/README.md +++ b/modules/bigtable-instance/README.md @@ -246,7 +246,7 @@ module "bigtable-instance" { | name | description | sensitive | |---|---|:---:| | [id](outputs.tf#L17) | Fully qualified instance id. | | -| [instance](outputs.tf#L26) | BigTable intance. | | +| [instance](outputs.tf#L26) | BigTable instance. | | | [table_ids](outputs.tf#L35) | Map of fully qualified table ids keyed by table name. | | | [tables](outputs.tf#L40) | Table resources. | | diff --git a/modules/projects-data-source/README.md b/modules/projects-data-source/README.md index 25402a04..93dd67f4 100644 --- a/modules/projects-data-source/README.md +++ b/modules/projects-data-source/README.md @@ -80,7 +80,7 @@ output "filtered-projects" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [parent](variables.tf#L55) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | ✓ | | -| [ignore_folders](variables.tf#L17) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | list(string) | | [] | +| [ignore_folders](variables.tf#L17) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are excluded from the output regardless of the include_projects variable. | list(string) | | [] | | [ignore_projects](variables.tf#L28) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | list(string) | | [] | | [include_projects](variables.tf#L41) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wildcard entries. | list(string) | | [] | | [query](variables.tf#L64) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | string | | "state:ACTIVE" | From c2c7ec4f1d012a339235b0a41fe76b620449f855 Mon Sep 17 00:00:00 2001 From: Alejandro Leal Date: Fri, 18 Aug 2023 06:35:09 +0000 Subject: [PATCH 24/46] Adding exception for idx --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ed88b00d..cbf110dd 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ **/.test.lock .idea .vscode +.idx/dev.nix backend.tf backend-config.hcl credentials.json From dc78ad3493b9e3322d943ab5a419e1ae7381c6ff Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 18 Aug 2023 10:06:32 +0200 Subject: [PATCH 25/46] Update outputs.tf --- modules/bigtable-instance/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bigtable-instance/outputs.tf b/modules/bigtable-instance/outputs.tf index 93f817dc..a2fd2646 100644 --- a/modules/bigtable-instance/outputs.tf +++ b/modules/bigtable-instance/outputs.tf @@ -24,7 +24,7 @@ output "id" { } output "instance" { - description = "BigTable intance." + description = "BigTable instance." value = google_bigtable_instance.default depends_on = [ google_bigtable_instance_iam_binding.default, From 476d6eaf307a027e6f69f28b9f031b84389e9bdc Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 18 Aug 2023 10:07:23 +0200 Subject: [PATCH 26/46] Update variables.tf --- modules/projects-data-source/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/projects-data-source/variables.tf b/modules/projects-data-source/variables.tf index c774784c..888cab21 100644 --- a/modules/projects-data-source/variables.tf +++ b/modules/projects-data-source/variables.tf @@ -15,7 +15,7 @@ */ variable "ignore_folders" { - description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable." + description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are excluded from the output regardless of the include_projects variable." type = list(string) default = [] # example exlusing a folder From 5a298780c1152ee6cf86c1113e84a4021db2bf2b Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 11:19:35 +0200 Subject: [PATCH 27/46] changed variable name --- tests/modules/gcve_private_cloud/examples/basic.yaml | 8 ++++---- tests/modules/gcve_private_cloud/examples/custom.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/modules/gcve_private_cloud/examples/basic.yaml b/tests/modules/gcve_private_cloud/examples/basic.yaml index b91dcf58..5c314b0e 100644 --- a/tests/modules/gcve_private_cloud/examples/basic.yaml +++ b/tests/modules/gcve_private_cloud/examples/basic.yaml @@ -15,14 +15,14 @@ values: module.gcve-pc.google_vmwareengine_network.private-cloud-network[0]: description: Terraform-managed. - location: asia-southeast1 - name: asia-southeast1-default + location: europe-west8 + name: europe-west8-default project: gcve-test-project timeouts: null type: LEGACY module.gcve-pc.google_vmwareengine_private_cloud.private-cloud: description: Terraform-managed. - location: asia-southeast1-a + location: europe-west8-a management_cluster: - cluster_id: gcve-pc-mgmt-cluster node_type_configs: @@ -31,7 +31,7 @@ values: node_type_id: standard-72 name: gcve-pc network_config: - - management_cidr: 192.168.0.0/24 + - cidr: 192.168.0.0/24 project: gcve-test-project timeouts: null diff --git a/tests/modules/gcve_private_cloud/examples/custom.yaml b/tests/modules/gcve_private_cloud/examples/custom.yaml index 4c813e2c..6869f4d8 100644 --- a/tests/modules/gcve_private_cloud/examples/custom.yaml +++ b/tests/modules/gcve_private_cloud/examples/custom.yaml @@ -15,14 +15,14 @@ values: module.gcve-pc.google_vmwareengine_network.private-cloud-network[0]: description: Terraform-managed. - location: asia-southeast1 - name: asia-southeast1-default + location: europe-west8 + name: europe-west8-default project: gcve-test-project timeouts: null type: LEGACY module.gcve-pc.google_vmwareengine_private_cloud.private-cloud: description: Terraform-managed. - location: asia-southeast1-a + location: europe-west8-a management_cluster: - cluster_id: gcve-pc-mgmt-cluster node_type_configs: @@ -31,7 +31,7 @@ values: node_type_id: standard-72 name: gcve-pc network_config: - - management_cidr: 192.168.0.0/24 + - cidr: 192.168.0.0/24 project: gcve-test-project timeouts: null From d568408331d6ae3b25c11ff9b7d88206693732e2 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 11:20:27 +0200 Subject: [PATCH 28/46] fixed variable name and regex --- modules/gcve-private-cloud/README.md | 32 ++++++++++++------------- modules/gcve-private-cloud/main.tf | 7 +++--- modules/gcve-private-cloud/outputs.tf | 24 +++++++++++-------- modules/gcve-private-cloud/variables.tf | 10 ++++---- 4 files changed, 40 insertions(+), 33 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index 0c3fe404..93e9271e 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -26,13 +26,13 @@ module "gcve-pc" { source = "./fabric/modules/gcve-private-cloud" name = "gcve-pc" project_id = "gcve-test-project" - zone = "asia-southeast1-a" - management_cidr = "192.168.0.0/24" + zone = "europe-west8-a" + cidr = "192.168.0.0/24" private_connections = { transit-conn1 = { name = "transit-conn1", - network_self_link = "projects/test-prj-elia-01/global/networks/default", + network_self_link = "projects/test-prj-gcve-01/global/networks/default", peering_name = "servicenetworking-googleapis-com" type = "PRIVATE_SERVICE_ACCESS", routing_mode = "REGIONAL" @@ -48,8 +48,8 @@ module "gcve-pc" { source = "./fabric/modules/gcve-private-cloud" name = "gcve-pc" project_id = "gcve-test-project" - zone = "asia-southeast1-a" - management_cidr = "192.168.0.0/24" + zone = "europe-west8-a" + cidr = "192.168.0.0/24" management_cluster_config = { node_type_id = "standard-72" @@ -60,7 +60,7 @@ module "gcve-pc" { private_connections = { transit-conn1 = { name = "transit-conn1", - network_self_link = "projects/test-prj-elia-01/global/networks/default", + network_self_link = "projects/test-prj-gcve-01/global/networks/default", peering_name = "servicenetworking-googleapis-com" type = "PRIVATE_SERVICE_ACCESS", routing_mode = "REGIONAL" @@ -74,15 +74,15 @@ module "gcve-pc" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [management_cidr](variables.tf#L23) | vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). | string | ✓ | | -| [name](variables.tf#L42) | Private cloud name. | string | ✓ | | -| [project_id](variables.tf#L74) | Project id. | string | ✓ | | -| [zone](variables.tf#L91) | Private cloud zone. | string | ✓ | | +| [cidr](variables.tf#L23) | vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). | string | ✓ | | +| [name](variables.tf#L43) | Private cloud name. | string | ✓ | | +| [project_id](variables.tf#L76) | Project id. | string | ✓ | | +| [zone](variables.tf#L93) | Private cloud zone. | string | ✓ | | | [description](variables.tf#L17) | Private cloud description. | string | | "Terraform-managed." | | [management_cluster_config](variables.tf#L28) | Management cluster configuration. | object({…}) | | {…} | -| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | -| [vmw_network_create](variables.tf#L79) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | -| [vmw_network_description](variables.tf#L85) | VMware Engine network description. | string | | "Terraform-managed." | +| [private_connections](variables.tf#L48) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [vmw_network_create](variables.tf#L81) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | +| [vmw_network_description](variables.tf#L87) | VMware Engine network description. | string | | "Terraform-managed." | ## Outputs @@ -94,7 +94,7 @@ module "gcve-pc" { | [network_config](outputs.tf#L32) | Details about the network configuration of the private cloud | | | [nsx](outputs.tf#L37) | Details about a NSX Manager appliance. | | | [private-cloud](outputs.tf#L42) | The private cloud resource | | -| [private_connections_setup](outputs.tf#L57) | Cloud SDK commands for the private connections manual setup. | | -| [state](outputs.tf#L52) | Details about the state of the private cloud | | -| [vcenter](outputs.tf#L47) | Details about a vCenter Server management appliance. | | +| [private_connections_setup](outputs.tf#L47) | Cloud SDK commands for the private connections manual setup. | | +| [state](outputs.tf#L63) | Details about the state of the private cloud | | +| [vcenter](outputs.tf#L68) | Details about a vCenter Server management appliance. | | diff --git a/modules/gcve-private-cloud/main.tf b/modules/gcve-private-cloud/main.tf index 13b915b7..875798d2 100644 --- a/modules/gcve-private-cloud/main.tf +++ b/modules/gcve-private-cloud/main.tf @@ -15,7 +15,7 @@ */ locals { - region = join("-", slice(split("-", "${var.zone}"), 0, 2)) + region = regex("([a-z]*-[a-z]*[0-9]{1,2})-([a-z])", var.zone)[0] vmw_network = ( var.vmw_network_create ? try(google_vmwareengine_network.private-cloud-network.0, null) @@ -23,7 +23,7 @@ locals { ) psa_peering = { for k, v in data.google_compute_network_peering.psa_peering : - k => slice(split("/", "${v.peer_network}"), 6, 7)[0] + k => regex("(.*)/projects/([a-z0-9-]*)/(.*)", "${v.peer_network}")[1] } } @@ -35,6 +35,7 @@ data "google_vmwareengine_network" "private-cloud-network" { location = local.region } +#TO REMOVE? data "google_compute_network_peering" "psa_peering" { for_each = var.private_connections @@ -50,7 +51,7 @@ resource "google_vmwareengine_private_cloud" "private-cloud" { description = var.description network_config { - management_cidr = var.management_cidr + management_cidr = var.cidr vmware_engine_network = local.vmw_network.id } diff --git a/modules/gcve-private-cloud/outputs.tf b/modules/gcve-private-cloud/outputs.tf index f1af67ec..ff60b349 100644 --- a/modules/gcve-private-cloud/outputs.tf +++ b/modules/gcve-private-cloud/outputs.tf @@ -44,16 +44,6 @@ output "private-cloud" { value = google_vmwareengine_private_cloud.private-cloud } -output "vcenter" { - description = "Details about a vCenter Server management appliance." - value = google_vmwareengine_private_cloud.private-cloud.vcenter -} - -output "state" { - description = "Details about the state of the private cloud" - value = google_vmwareengine_private_cloud.private-cloud.state -} - output "private_connections_setup" { description = "Cloud SDK commands for the private connections manual setup." value = { @@ -69,3 +59,17 @@ output "private_connections_setup" { EOT } } + +output "state" { + description = "Details about the state of the private cloud" + value = google_vmwareengine_private_cloud.private-cloud.state +} + +output "vcenter" { + description = "Details about a vCenter Server management appliance." + value = google_vmwareengine_private_cloud.private-cloud.vcenter +} + +output "test" { + value = data.google_compute_network_peering.psa_peering +} diff --git a/modules/gcve-private-cloud/variables.tf b/modules/gcve-private-cloud/variables.tf index 945b5b10..1449476b 100644 --- a/modules/gcve-private-cloud/variables.tf +++ b/modules/gcve-private-cloud/variables.tf @@ -20,7 +20,7 @@ variable "description" { default = "Terraform-managed." } -variable "management_cidr" { +variable "cidr" { description = "vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements)." type = string } @@ -37,6 +37,7 @@ variable "management_cluster_config" { node_count = 3, custom_core_count = null } + nullable = false } variable "name" { @@ -51,10 +52,11 @@ variable "private_connections" { network_self_link = string peering_name = string description = optional(string, "Terraform-managed.") - type = optional(string, "REGIONAL") - routing_mode = optional(string, "PRIVATE_SERVICE_ACCESS") + type = optional(string, "PRIVATE_SERVICE_ACCESS") + routing_mode = optional(string, "REGIONAL") })) - default = {} + default = {} + nullable = false validation { condition = alltrue([ for r in var.private_connections : From e1e3826f6d76fd256da1704f901928ddb3fb346a Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 18 Aug 2023 13:45:42 +0100 Subject: [PATCH 29/46] fix(cloud-run): move cpu boost annotation to revision --- modules/cloud-run/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/cloud-run/main.tf b/modules/cloud-run/main.tf index 617e67bb..527e1fd1 100644 --- a/modules/cloud-run/main.tf +++ b/modules/cloud-run/main.tf @@ -34,9 +34,6 @@ locals { var.ingress_settings == null ? {} : { "run.googleapis.com/ingress" = var.ingress_settings }, - var.startup_cpu_boost ? { - "run.googleapis.com/startup-cpu-boost" = "true" - } : {} ) _iam_run_invoker_members = concat( lookup(var.iam, "roles/run.invoker", []), @@ -75,6 +72,9 @@ locals { var.gen2_execution_environment ? { "run.googleapis.com/execution-environment" = "gen2" } : {}, + var.startup_cpu_boost ? { + "run.googleapis.com/startup-cpu-boost" = "true" + } : {}, ) revision_name = ( try(var.revision_name, null) == null From e762e93677160daf8c3a14602d4e6c882f093cc6 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 15:37:32 +0200 Subject: [PATCH 30/46] fixed variable names --- tests/modules/gcve_private_cloud/examples/basic.yaml | 2 +- tests/modules/gcve_private_cloud/examples/custom.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/gcve_private_cloud/examples/basic.yaml b/tests/modules/gcve_private_cloud/examples/basic.yaml index 5c314b0e..ec6544f6 100644 --- a/tests/modules/gcve_private_cloud/examples/basic.yaml +++ b/tests/modules/gcve_private_cloud/examples/basic.yaml @@ -31,7 +31,7 @@ values: node_type_id: standard-72 name: gcve-pc network_config: - - cidr: 192.168.0.0/24 + - management_cidr: 192.168.0.0/24 project: gcve-test-project timeouts: null diff --git a/tests/modules/gcve_private_cloud/examples/custom.yaml b/tests/modules/gcve_private_cloud/examples/custom.yaml index 6869f4d8..2a413d18 100644 --- a/tests/modules/gcve_private_cloud/examples/custom.yaml +++ b/tests/modules/gcve_private_cloud/examples/custom.yaml @@ -31,7 +31,7 @@ values: node_type_id: standard-72 name: gcve-pc network_config: - - cidr: 192.168.0.0/24 + - management_cidr: 192.168.0.0/24 project: gcve-test-project timeouts: null From 87e82244afa777f646f9975cf28e005793acabb9 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 15:38:35 +0200 Subject: [PATCH 31/46] fixed tests --- modules/gcve-private-cloud/README.md | 51 +++++++++++++------------ modules/gcve-private-cloud/main.tf | 11 +++--- modules/gcve-private-cloud/outputs.tf | 2 +- modules/gcve-private-cloud/variables.tf | 21 +++++++--- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index 93e9271e..a0feefdd 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -23,19 +23,19 @@ At the moment this module doesn't support the following use cases: ```hcl module "gcve-pc" { - source = "./fabric/modules/gcve-private-cloud" - name = "gcve-pc" - project_id = "gcve-test-project" - zone = "europe-west8-a" - cidr = "192.168.0.0/24" + source = "./fabric/modules/gcve-private-cloud" + name = "gcve-pc" + project_id = "gcve-test-project" + zone = "europe-west8-a" + cidr = "192.168.0.0/24" private_connections = { transit-conn1 = { - name = "transit-conn1", - network_self_link = "projects/test-prj-gcve-01/global/networks/default", - peering_name = "servicenetworking-googleapis-com" - type = "PRIVATE_SERVICE_ACCESS", - routing_mode = "REGIONAL" + name = "transit-conn1" + network_self_link = "projects/test-prj-gcve-01/global/networks/default" + tenant_host_project = "g39a814990532d10ap-tp" + type = "PRIVATE_SERVICE_ACCESS" + routing_mode = "REGIONAL" } } } @@ -45,11 +45,11 @@ module "gcve-pc" { ```hcl module "gcve-pc" { - source = "./fabric/modules/gcve-private-cloud" - name = "gcve-pc" - project_id = "gcve-test-project" - zone = "europe-west8-a" - cidr = "192.168.0.0/24" + source = "./fabric/modules/gcve-private-cloud" + name = "gcve-pc" + project_id = "gcve-test-project" + zone = "europe-west8-a" + cidr = "192.168.0.0/24" management_cluster_config = { node_type_id = "standard-72" @@ -59,11 +59,11 @@ module "gcve-pc" { private_connections = { transit-conn1 = { - name = "transit-conn1", - network_self_link = "projects/test-prj-gcve-01/global/networks/default", - peering_name = "servicenetworking-googleapis-com" - type = "PRIVATE_SERVICE_ACCESS", - routing_mode = "REGIONAL" + name = "transit-conn1" + network_self_link = "projects/test-prj-gcve-01/global/networks/default" + tenant_host_project = "g39a814990532d10ap-tp" + type = "PRIVATE_SERVICE_ACCESS" + routing_mode = "REGIONAL" } } } @@ -76,13 +76,13 @@ module "gcve-pc" { |---|---|:---:|:---:|:---:| | [cidr](variables.tf#L23) | vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). | string | ✓ | | | [name](variables.tf#L43) | Private cloud name. | string | ✓ | | -| [project_id](variables.tf#L76) | Project id. | string | ✓ | | -| [zone](variables.tf#L93) | Private cloud zone. | string | ✓ | | +| [project_id](variables.tf#L85) | Project id. | string | ✓ | | +| [zone](variables.tf#L102) | Private cloud zone. | string | ✓ | | | [description](variables.tf#L17) | Private cloud description. | string | | "Terraform-managed." | | [management_cluster_config](variables.tf#L28) | Management cluster configuration. | object({…}) | | {…} | -| [private_connections](variables.tf#L48) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | -| [vmw_network_create](variables.tf#L81) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | -| [vmw_network_description](variables.tf#L87) | VMware Engine network description. | string | | "Terraform-managed." | +| [private_connections](variables.tf#L48) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [vmw_network_create](variables.tf#L90) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | +| [vmw_network_description](variables.tf#L96) | VMware Engine network description. | string | | "Terraform-managed." | ## Outputs @@ -96,5 +96,6 @@ module "gcve-pc" { | [private-cloud](outputs.tf#L42) | The private cloud resource | | | [private_connections_setup](outputs.tf#L47) | Cloud SDK commands for the private connections manual setup. | | | [state](outputs.tf#L63) | Details about the state of the private cloud | | +| [test](outputs.tf#L73) | | | | [vcenter](outputs.tf#L68) | Details about a vCenter Server management appliance. | | diff --git a/modules/gcve-private-cloud/main.tf b/modules/gcve-private-cloud/main.tf index 875798d2..3235fb29 100644 --- a/modules/gcve-private-cloud/main.tf +++ b/modules/gcve-private-cloud/main.tf @@ -21,9 +21,10 @@ locals { ? try(google_vmwareengine_network.private-cloud-network.0, null) : try(data.google_vmwareengine_network.private-cloud-network.0, null) ) - psa_peering = { - for k, v in data.google_compute_network_peering.psa_peering : - k => regex("(.*)/projects/([a-z0-9-]*)/(.*)", "${v.peer_network}")[1] + tenant_host_project = { + for k, v in var.private_connections : k => v.tenant_host_project == null + ? regex("(.*)/projects/([a-z0-9-]*)/(.*)", "${data.google_compute_network_peering.psa_peering[k].peer_network}")[1] + : v.tenant_host_project } } @@ -35,10 +36,8 @@ data "google_vmwareengine_network" "private-cloud-network" { location = local.region } -#TO REMOVE? - data "google_compute_network_peering" "psa_peering" { - for_each = var.private_connections + for_each = { for k, v in var.private_connections : k => v if v.tenant_host_project == null } name = each.value.peering_name network = each.value.network_self_link } diff --git a/modules/gcve-private-cloud/outputs.tf b/modules/gcve-private-cloud/outputs.tf index ff60b349..76267f21 100644 --- a/modules/gcve-private-cloud/outputs.tf +++ b/modules/gcve-private-cloud/outputs.tf @@ -54,7 +54,7 @@ output "private_connections_setup" { --vmware-engine-network=${local.region}-default \ --description="${v.description}" \ --routing-mode=${v.routing_mode} \ - --service-project=${local.psa_peering[k]} \ + --service-project=${local.tenant_host_project[k]} \ --type=${v.type} EOT } diff --git a/modules/gcve-private-cloud/variables.tf b/modules/gcve-private-cloud/variables.tf index 1449476b..d81eb0b4 100644 --- a/modules/gcve-private-cloud/variables.tf +++ b/modules/gcve-private-cloud/variables.tf @@ -48,15 +48,24 @@ variable "name" { variable "private_connections" { description = "VMWare private connections configuration. It is used to create the gcloud command printed as output." type = map(object({ - name = string - network_self_link = string - peering_name = string - description = optional(string, "Terraform-managed.") - type = optional(string, "PRIVATE_SERVICE_ACCESS") - routing_mode = optional(string, "REGIONAL") + name = string + network_self_link = string + peering_name = optional(string) + tenant_host_project = optional(string) + description = optional(string, "Terraform-managed.") + type = optional(string, "PRIVATE_SERVICE_ACCESS") + routing_mode = optional(string, "REGIONAL") })) default = {} nullable = false + validation { + condition = alltrue([ + for k, v in var.private_connections : + (v.peering_name != null) != (v.tenant_host_project != null) + ] + ) + error_message = "Both peering_name and tenant_host_project variables have been set. Only one variable is allowed." + } validation { condition = alltrue([ for r in var.private_connections : From 81b567684b6d2450cac17062a42f7ebd28fd26dc Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 15:41:06 +0200 Subject: [PATCH 32/46] outputs cleaup --- modules/gcve-private-cloud/outputs.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/gcve-private-cloud/outputs.tf b/modules/gcve-private-cloud/outputs.tf index 76267f21..15ca17af 100644 --- a/modules/gcve-private-cloud/outputs.tf +++ b/modules/gcve-private-cloud/outputs.tf @@ -69,7 +69,3 @@ output "vcenter" { description = "Details about a vCenter Server management appliance." value = google_vmwareengine_private_cloud.private-cloud.vcenter } - -output "test" { - value = data.google_compute_network_peering.psa_peering -} From 9452a14ac75f17ace5866faccad10be9e6b43879 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 15:49:20 +0200 Subject: [PATCH 33/46] output cleanup --- modules/gcve-private-cloud/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index a0feefdd..ce0c710a 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -96,6 +96,5 @@ module "gcve-pc" { | [private-cloud](outputs.tf#L42) | The private cloud resource | | | [private_connections_setup](outputs.tf#L47) | Cloud SDK commands for the private connections manual setup. | | | [state](outputs.tf#L63) | Details about the state of the private cloud | | -| [test](outputs.tf#L73) | | | | [vcenter](outputs.tf#L68) | Details about a vCenter Server management appliance. | | From bf26580b9da52f72d1c72e4f50661c7a67398218 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 15:52:56 +0200 Subject: [PATCH 34/46] fixed variables order --- modules/gcve-private-cloud/variables.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/gcve-private-cloud/variables.tf b/modules/gcve-private-cloud/variables.tf index d81eb0b4..934b0efe 100644 --- a/modules/gcve-private-cloud/variables.tf +++ b/modules/gcve-private-cloud/variables.tf @@ -13,6 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +variable "cidr" { + description = "vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements)." + type = string +} variable "description" { description = "Private cloud description." @@ -20,11 +24,6 @@ variable "description" { default = "Terraform-managed." } -variable "cidr" { - description = "vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements)." - type = string -} - variable "management_cluster_config" { description = "Management cluster configuration." type = object({ From c4ada40275d28d3e35f8129fc8d61b7fea193f52 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 16:52:58 +0200 Subject: [PATCH 35/46] fixed typo --- modules/gcve-private-cloud/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index ce0c710a..b54cc173 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -74,15 +74,15 @@ module "gcve-pc" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [cidr](variables.tf#L23) | vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). | string | ✓ | | -| [name](variables.tf#L43) | Private cloud name. | string | ✓ | | -| [project_id](variables.tf#L85) | Project id. | string | ✓ | | -| [zone](variables.tf#L102) | Private cloud zone. | string | ✓ | | -| [description](variables.tf#L17) | Private cloud description. | string | | "Terraform-managed." | -| [management_cluster_config](variables.tf#L28) | Management cluster configuration. | object({…}) | | {…} | -| [private_connections](variables.tf#L48) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | -| [vmw_network_create](variables.tf#L90) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | -| [vmw_network_description](variables.tf#L96) | VMware Engine network description. | string | | "Terraform-managed." | +| [cidr](variables.tf#L16) | vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). | string | ✓ | | +| [name](variables.tf#L42) | Private cloud name. | string | ✓ | | +| [project_id](variables.tf#L84) | Project id. | string | ✓ | | +| [zone](variables.tf#L101) | Private cloud zone. | string | ✓ | | +| [description](variables.tf#L21) | Private cloud description. | string | | "Terraform-managed." | +| [management_cluster_config](variables.tf#L27) | Management cluster configuration. | object({…}) | | {…} | +| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [vmw_network_create](variables.tf#L89) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | +| [vmw_network_description](variables.tf#L95) | VMware Engine network description. | string | | "Terraform-managed." | ## Outputs From 1b93197b87004534c69ae27b0080bf7c942be382 Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 16:56:21 +0200 Subject: [PATCH 36/46] fixed typo --- modules/gcve-private-cloud/outputs.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gcve-private-cloud/outputs.tf b/modules/gcve-private-cloud/outputs.tf index 15ca17af..0b7280ce 100644 --- a/modules/gcve-private-cloud/outputs.tf +++ b/modules/gcve-private-cloud/outputs.tf @@ -40,7 +40,7 @@ output "nsx" { } output "private-cloud" { - description = "The private cloud resource" + description = "The private cloud resource." value = google_vmwareengine_private_cloud.private-cloud } @@ -61,7 +61,7 @@ output "private_connections_setup" { } output "state" { - description = "Details about the state of the private cloud" + description = "Details about the state of the private clou." value = google_vmwareengine_private_cloud.private-cloud.state } From 103443dc3036204704a79ce0096e0b046b3b96de Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 16:57:35 +0200 Subject: [PATCH 37/46] fixed typo --- modules/gcve-private-cloud/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index b54cc173..5d5bde31 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -93,8 +93,8 @@ module "gcve-pc" { | [management_cluster](outputs.tf#L27) | Details of the management cluster of the private cloud | | | [network_config](outputs.tf#L32) | Details about the network configuration of the private cloud | | | [nsx](outputs.tf#L37) | Details about a NSX Manager appliance. | | -| [private-cloud](outputs.tf#L42) | The private cloud resource | | +| [private-cloud](outputs.tf#L42) | The private cloud resource. | | | [private_connections_setup](outputs.tf#L47) | Cloud SDK commands for the private connections manual setup. | | -| [state](outputs.tf#L63) | Details about the state of the private cloud | | +| [state](outputs.tf#L63) | Details about the state of the private clou. | | | [vcenter](outputs.tf#L68) | Details about a vCenter Server management appliance. | | From 126d75a311dd6256dabdb1562bf21ab0f669208b Mon Sep 17 00:00:00 2001 From: eliamaldini Date: Fri, 18 Aug 2023 17:01:12 +0200 Subject: [PATCH 38/46] fixed typo --- modules/gcve-private-cloud/README.md | 8 ++++---- modules/gcve-private-cloud/outputs.tf | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/gcve-private-cloud/README.md b/modules/gcve-private-cloud/README.md index 5d5bde31..901b3079 100644 --- a/modules/gcve-private-cloud/README.md +++ b/modules/gcve-private-cloud/README.md @@ -89,12 +89,12 @@ module "gcve-pc" { | name | description | sensitive | |---|---|:---:| | [hcx](outputs.tf#L17) | Details about a HCX Cloud Manager appliance. | | -| [id](outputs.tf#L22) | ID of the private cloud | | -| [management_cluster](outputs.tf#L27) | Details of the management cluster of the private cloud | | -| [network_config](outputs.tf#L32) | Details about the network configuration of the private cloud | | +| [id](outputs.tf#L22) | ID of the private cloud. | | +| [management_cluster](outputs.tf#L27) | Details of the management cluster of the private cloud. | | +| [network_config](outputs.tf#L32) | Details about the network configuration of the private cloud. | | | [nsx](outputs.tf#L37) | Details about a NSX Manager appliance. | | | [private-cloud](outputs.tf#L42) | The private cloud resource. | | | [private_connections_setup](outputs.tf#L47) | Cloud SDK commands for the private connections manual setup. | | -| [state](outputs.tf#L63) | Details about the state of the private clou. | | +| [state](outputs.tf#L63) | Details about the state of the private cloud. | | | [vcenter](outputs.tf#L68) | Details about a vCenter Server management appliance. | | diff --git a/modules/gcve-private-cloud/outputs.tf b/modules/gcve-private-cloud/outputs.tf index 0b7280ce..55578f6d 100644 --- a/modules/gcve-private-cloud/outputs.tf +++ b/modules/gcve-private-cloud/outputs.tf @@ -20,17 +20,17 @@ output "hcx" { } output "id" { - description = "ID of the private cloud" + description = "ID of the private cloud." value = google_vmwareengine_private_cloud.private-cloud.id } output "management_cluster" { - description = "Details of the management cluster of the private cloud" + description = "Details of the management cluster of the private cloud." value = google_vmwareengine_private_cloud.private-cloud.management_cluster } output "network_config" { - description = "Details about the network configuration of the private cloud" + description = "Details about the network configuration of the private cloud." value = google_vmwareengine_private_cloud.private-cloud.network_config } @@ -61,7 +61,7 @@ output "private_connections_setup" { } output "state" { - description = "Details about the state of the private clou." + description = "Details about the state of the private cloud." value = google_vmwareengine_private_cloud.private-cloud.state } From 6eeba5e599b62010cb46503103346f3433769c0c Mon Sep 17 00:00:00 2001 From: lcaggio Date: Fri, 18 Aug 2023 18:27:43 +0200 Subject: [PATCH 39/46] [Data Platform] Update README.md (#1601) Fix hardcoded path in readme. --- fast/stages/3-data-platform/dev/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index 2d1e5597..9836b598 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -95,12 +95,12 @@ The commands to link or copy the provider and terraform variable files can be ea # copy and paste the following commands for '3-data-platform' -ln -s /home/ludomagno/fast-config/providers/3-data-platform-providers.tf ./ -ln -s /home/ludomagno/fast-config/tfvars/globals.auto.tfvars.json ./ -ln -s /home/ludomagno/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./ -ln -s /home/ludomagno/fast-config/tfvars/1-resman.auto.tfvars.json ./ -ln -s /home/ludomagno/fast-config/tfvars/2-networking.auto.tfvars.json ./ -ln -s /home/ludomagno/fast-config/tfvars/2-security.auto.tfvars.json ./ +ln -s ~/fast-config/providers/3-data-platform-providers.tf ./ +ln -s ~/fast-config/tfvars/globals.auto.tfvars.json ./ +ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./ +ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json ./ +ln -s ~/fast-config/tfvars/2-networking.auto.tfvars.json ./ +ln -s ~/fast-config/tfvars/2-security.auto.tfvars.json ./ ``` ```bash From 819894d2bab4b440f1b52b1ac8035912fb107004 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 20 Aug 2023 09:44:20 +0200 Subject: [PATCH 40/46] IAM interface refactor (#1595) * IAM modules refactor proposal * policy * subheading * Update 20230816-iam-refactor.md * log Julio's +1 * data-catalog-policy-tag * dataproc * dataproc * folder * folder * folder * folder * project * better filtering in test examples * project * folder * folder * organization * fix variable descriptions * kms * net-vpc * dataplex-datascan * modules/iam-service-account * modules/source-repository/ * blueprints/cloud-operations/vm-migration/ * blueprints/third-party-solutions/wordpress * dataplex-datascan * blueprints/cloud-operations/workload-identity-federation * blueprints/data-solutions/cloudsql-multiregion/ * blueprints/data-solutions/composer-2 * Update 20230816-iam-refactor.md * Update 20230816-iam-refactor.md * capture discussion in architectural doc * update variable names and refactor proposal * project * blueprints first round * folder * organization * data-catalog-policy-tag * re-enable folder inventory * project module style fix * dataproc * source-repository * source-repository tests * dataplex-datascan * dataplex-datascan tests * net-vpc * net-vpc test examples * iam-service-account * iam-service-account test examples * kms * boilerplate * tfdoc * fix module tests * more blueprint fixes * fix typo in data blueprints * incomplete refactor of data platform foundations * tfdoc * data platform foundation * refactor data platform foundation iam locals * remove redundant example test * shielded folder fix * fix typo * project factory * project factory outputs * tfdoc * test workflow: less verbose tests, fix tf version * re-enable -vv, shorter traceback, fix action version * ignore github extension warning, re-enable action version * fast bootstrap IAM, untested * bootstrap stage IAM fixes * stage 0 tests * fast stage 1 * tenant stage 1 * minor changes to fast stage 0 and 1 * fast security stage * fast mt stage 0 * fast mt stage 0 * fast pf --- .github/workflows/daily-tag.yml | 2 +- .github/workflows/linting.yml | 3 - .github/workflows/tests.yml | 25 +- .../host-target-projects/README.md | 19 +- .../vm-migration/host-target-projects/main.tf | 47 ++- .../host-target-projects/variables.tf | 14 +- .../host-target-sharedvpc/README.md | 27 +- .../host-target-sharedvpc/main.tf | 49 +-- .../host-target-sharedvpc/variables.tf | 15 +- .../vm-migration/single-project/README.md | 17 +- .../vm-migration/single-project/main.tf | 25 +- .../vm-migration/single-project/variables.tf | 14 +- .../workload-identity-federation/README.md | 16 + .../google-cloud.tf | 7 +- .../cloudsql-multiregion/README.md | 25 +- .../cloudsql-multiregion/cloudsql.tf | 5 +- .../cloudsql-multiregion/datastorage.tf | 30 -- .../cloudsql-multiregion/main.tf | 86 +++-- .../cloudsql-multiregion/outputs.tf | 9 +- .../cloudsql-multiregion/variables.tf | 14 +- .../data-solutions/composer-2/README.md | 72 +++-- blueprints/data-solutions/composer-2/main.tf | 78 ++--- .../data-solutions/composer-2/variables.tf | 7 +- .../data-platform-foundations/01-dropoff.tf | 49 +-- .../data-platform-foundations/02-load.tf | 54 ++-- .../03-orchestration.tf | 87 ++--- .../04-transformation.tf | 52 ++- .../05-datawarehouse.tf | 131 ++++---- .../data-platform-foundations/06-common.tf | 62 ++-- .../data-platform-foundations/07-exposure.tf | 8 +- .../data-platform-foundations/README.md | 47 +-- .../locals-01-dropoff.tf | 37 +++ .../locals-02-load.tf | 47 +++ .../locals-03-orchestration.tf | 50 +++ .../locals-04-transformation.tf | 47 +++ .../locals-05-datawarehouse.tf | 68 ++++ .../locals-06-common.tf | 37 +++ .../data-platform-foundations/main.tf | 17 + .../data-platform-foundations/variables.tf | 101 +++--- .../data-platform-minimal/01-landing.tf | 35 +- .../data-platform-minimal/02-processing.tf | 61 +++- .../data-platform-minimal/03-curated.tf | 66 ++-- .../data-platform-minimal/04-common.tf | 41 ++- .../data-platform-minimal/README.md | 28 +- .../gcs-to-bq-with-least-privileges/main.tf | 39 ++- .../data-solutions/shielded-folder/README.md | 20 +- .../data-solutions/shielded-folder/kms.tf | 23 +- .../shielded-folder/variables.tf | 9 +- .../data-solutions/sqlserver-alwayson/main.tf | 3 - .../factories/project-factory/README.md | 305 ++++-------------- .../factories/project-factory/factory.tf | 100 ++++++ blueprints/factories/project-factory/main.tf | 268 ++++----------- .../factories/project-factory/outputs.tf | 28 +- .../project-factory/sample-data/defaults.yaml | 29 -- .../sample-data/projects/project.yaml | 117 ------- .../factories/project-factory/variables.tf | 271 ++++------------ .../networking/hub-and-spoke-peering/main.tf | 8 +- blueprints/networking/shared-vpc-gke/main.tf | 8 +- .../wordpress/cloudrun/README.md | 46 ++- .../wordpress/cloudrun/main.tf | 93 +++--- .../wordpress/cloudrun/variables.tf | 12 +- .../0-bootstrap-tenant/README.md | 29 +- .../0-bootstrap-tenant/automation-sas.tf | 1 + .../0-bootstrap-tenant/cicd.tf | 12 +- .../0-bootstrap-tenant/main.tf | 4 +- .../0-bootstrap-tenant/organization.tf | 67 ++-- .../0-bootstrap-tenant/variables.tf | 30 +- .../1-resman-tenant/root_node.tf | 23 +- fast/stages/0-bootstrap/README.md | 37 ++- fast/stages/0-bootstrap/automation.tf | 17 +- fast/stages/0-bootstrap/organization-iam.tf | 153 +++++++++ fast/stages/0-bootstrap/organization.tf | 249 +++++--------- fast/stages/0-bootstrap/variables.tf | 41 ++- fast/stages/1-resman/README.md | 5 +- .../stages/1-resman/branch-project-factory.tf | 30 -- fast/stages/1-resman/branch-tenants.tf | 25 +- fast/stages/1-resman/organization-iam.tf | 141 ++++++++ fast/stages/1-resman/organization.tf | 114 +++---- fast/stages/2-security/core-dev.tf | 2 +- fast/stages/2-security/core-prod.tf | 2 +- fast/stages/3-project-factory/dev/README.md | 56 +--- fast/stages/3-project-factory/dev/main.tf | 53 +-- fast/stages/3-project-factory/dev/outputs.tf | 9 +- .../stages/3-project-factory/dev/variables.tf | 45 +-- modules/__docs/20230816-iam-refactor.md | 148 +++++++++ modules/__docs/README.md | 3 + modules/cloud-run/README.md | 3 +- modules/data-catalog-policy-tag/README.md | 30 +- modules/data-catalog-policy-tag/iam.tf | 39 +-- modules/data-catalog-policy-tag/variables.tf | 26 +- modules/dataplex-datascan/README.md | 38 +-- modules/dataplex-datascan/iam.tf | 60 +--- modules/dataplex-datascan/variables.tf | 33 +- modules/dataproc/README.md | 67 ++-- modules/dataproc/iam.tf | 47 ++- modules/dataproc/variables.tf | 21 +- modules/folder/README.md | 76 ++--- modules/folder/iam.tf | 71 +--- modules/folder/logging.tf | 8 +- modules/folder/outputs.tf | 5 +- modules/folder/variables.tf | 34 +- modules/iam-service-account/README.md | 34 +- modules/iam-service-account/iam.tf | 63 ++-- modules/iam-service-account/variables.tf | 35 +- modules/kms/README.md | 50 ++- modules/kms/iam.tf | 78 ++--- modules/kms/outputs.tf | 18 +- modules/kms/variables.tf | 40 ++- modules/net-vpc/README.md | 68 ++-- modules/net-vpc/outputs.tf | 5 - modules/net-vpc/subnets.tf | 68 ++-- modules/net-vpc/variables.tf | 21 +- modules/organization/README.md | 78 ++--- modules/organization/iam.tf | 71 +--- modules/organization/logging.tf | 13 +- modules/organization/organization-policies.tf | 4 +- modules/organization/outputs.tf | 8 +- modules/organization/tags.tf | 4 +- modules/organization/variables.tf | 34 +- modules/project/README.md | 236 +++++--------- modules/project/iam.tf | 112 +------ modules/project/logging.tf | 11 +- modules/project/main.tf | 15 +- modules/project/variables.tf | 72 ++--- modules/source-repository/README.md | 13 +- modules/source-repository/iam.tf | 38 +-- modules/source-repository/variables.tf | 27 +- .../project_factory/examples/example.yaml | 297 +++++------------ tests/examples/conftest.py | 2 +- tests/fast/stages/s0_bootstrap/simple.yaml | 2 +- .../stages/s3_project_factory/common.tfvars | 8 +- .../s3_project_factory/data/defaults.yaml | 36 --- .../data/projects/project.yaml | 94 +----- .../examples/datascan_iam.yaml | 2 +- tests/modules/folder/examples/iam-policy.yaml | 27 -- tests/modules/folder/examples/iam.yaml | 27 +- .../iam_service_account/examples/basic.yaml | 2 +- tests/modules/net_vpc/examples/factory.yaml | 74 ++++- .../modules/net_vpc/examples/shared-vpc.yaml | 4 +- .../modules/net_vpc/examples/subnet-iam.yaml | 29 +- tests/modules/net_vpc/psa_routes_export.yaml | 1 - tests/modules/net_vpc/psa_routes_import.yaml | 1 - .../net_vpc/psa_routes_import_export.yaml | 1 - tests/modules/net_vpc/shared_vpc.yaml | 1 - .../modules/organization/examples/basic.yaml | 16 +- .../organization/examples/iam-policy.yaml | 23 -- .../test_plan_org_policies_modules.py | 4 +- .../examples/iam-additive-members.yaml | 33 -- .../project/examples/iam-additive.yaml | 36 --- .../project/examples/iam-authoritative.yaml | 16 +- .../examples/iam-bindings-additive.yaml | 46 +++ .../{iam-members.yaml => iam-bindings.yaml} | 47 ++- .../modules/project/examples/iam-policy.yaml | 34 -- .../source_repository/examples/simple.yaml | 3 +- 154 files changed, 3273 insertions(+), 3773 deletions(-) delete mode 100644 blueprints/data-solutions/cloudsql-multiregion/datastorage.tf create mode 100644 blueprints/data-solutions/data-platform-foundations/locals-01-dropoff.tf create mode 100644 blueprints/data-solutions/data-platform-foundations/locals-02-load.tf create mode 100644 blueprints/data-solutions/data-platform-foundations/locals-03-orchestration.tf create mode 100644 blueprints/data-solutions/data-platform-foundations/locals-04-transformation.tf create mode 100644 blueprints/data-solutions/data-platform-foundations/locals-05-datawarehouse.tf create mode 100644 blueprints/data-solutions/data-platform-foundations/locals-06-common.tf create mode 100644 blueprints/factories/project-factory/factory.tf delete mode 100644 blueprints/factories/project-factory/sample-data/defaults.yaml delete mode 100644 blueprints/factories/project-factory/sample-data/projects/project.yaml create mode 100644 fast/stages/0-bootstrap/organization-iam.tf create mode 100644 fast/stages/1-resman/organization-iam.tf create mode 100644 modules/__docs/20230816-iam-refactor.md create mode 100644 modules/__docs/README.md delete mode 100644 tests/fast/stages/s3_project_factory/data/defaults.yaml delete mode 100644 tests/modules/folder/examples/iam-policy.yaml delete mode 100644 tests/modules/organization/examples/iam-policy.yaml delete mode 100644 tests/modules/project/examples/iam-additive-members.yaml delete mode 100644 tests/modules/project/examples/iam-additive.yaml create mode 100644 tests/modules/project/examples/iam-bindings-additive.yaml rename tests/modules/project/examples/{iam-members.yaml => iam-bindings.yaml} (54%) delete mode 100644 tests/modules/project/examples/iam-policy.yaml diff --git a/.github/workflows/daily-tag.yml b/.github/workflows/daily-tag.yml index 7a0efc59..99424ad6 100644 --- a/.github/workflows/daily-tag.yml +++ b/.github/workflows/daily-tag.yml @@ -28,7 +28,7 @@ jobs: name: "Create tag on master if there was activity in last 24 hours" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: "Check changes and tag" run: | diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 815e1b2f..035b82fe 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -17,9 +17,6 @@ on: pull_request: branches: - master - tags: - - ci - - lint jobs: linting: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9fce75d1..de0a6694 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,9 +19,6 @@ on: pull_request: branches: - master - tags: - - ci - - test env: GOOGLE_APPLICATION_CREDENTIALS: "/home/runner/credentials.json" @@ -39,7 +36,7 @@ jobs: - uses: hashicorp/setup-terraform@v2 with: - terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_version: ${{ env.TF_VERSION }} terraform_wrapper: false - name: Build lockfile and fetch providers @@ -76,10 +73,10 @@ jobs: uses: ./.github/actions/fabric-tests with: PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }} + TERRAFORM_VERSION: ${{ env.TF_VERSION }} - name: Run tests on documentation examples - run: pytest -vv -n4 -k blueprints/ tests/examples + run: pytest -vv -n4 --tb=line -k blueprints/ tests/examples examples-modules: runs-on: ubuntu-latest @@ -91,10 +88,10 @@ jobs: uses: ./.github/actions/fabric-tests with: PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }} + TERRAFORM_VERSION: ${{ env.TF_VERSION }} - name: Run tests on documentation examples - run: pytest -vv -n4 -k modules/ tests/examples + run: pytest -vv -n4 --tb=line -k modules/ tests/examples blueprints: runs-on: ubuntu-latest @@ -106,10 +103,10 @@ jobs: uses: ./.github/actions/fabric-tests with: PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }} + TERRAFORM_VERSION: ${{ env.TF_VERSION }} - name: Run tests environments - run: pytest -vv -n4 tests/blueprints + run: pytest -vv -n4 --tb=line tests/blueprints modules: runs-on: ubuntu-latest @@ -121,10 +118,10 @@ jobs: uses: ./.github/actions/fabric-tests with: PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }} + TERRAFORM_VERSION: ${{ env.TF_VERSION }} - name: Run tests modules - run: pytest -vv -n4 tests/modules + run: pytest -vv -n4 --tb=line tests/modules fast: runs-on: ubuntu-latest @@ -136,7 +133,7 @@ jobs: uses: ./.github/actions/fabric-tests with: PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }} + TERRAFORM_VERSION: ${{ env.TF_VERSION }} - name: Run tests on FAST stages - run: pytest -vv -n4 tests/fast + run: pytest -vv -n4 --tb=line tests/fast diff --git a/blueprints/cloud-operations/vm-migration/host-target-projects/README.md b/blueprints/cloud-operations/vm-migration/host-target-projects/README.md index 92c35a2b..3168a1cd 100644 --- a/blueprints/cloud-operations/vm-migration/host-target-projects/README.md +++ b/blueprints/cloud-operations/vm-migration/host-target-projects/README.md @@ -13,21 +13,20 @@ This is the high level diagram: This sample creates\updates several distinct groups of resources: - projects - - Deploy M4CE host project with [required services](https://cloud.google.com/migrate/compute-engine/docs/5.0/how-to/enable-services#enabling_required_services_on_the_host_project) on a new or existing project. - - M4CE target project prerequisites deployed on existing projects. + - Deploy M4CE host project with [required services](https://cloud.google.com/migrate/compute-engine/docs/5.0/how-to/enable-services#enabling_required_services_on_the_host_project) on a new or existing project. + - M4CE target project prerequisites deployed on existing projects. - IAM - Create a [service account](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/migrate-connector#step-3) used at runtime by the M4CE connector for data replication - - Grant [migration admin roles](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to provided user accounts - - Grant [migration viewer role](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to provided user accounts + - Grant [migration admin roles](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to provided user or group + - Grant [migration viewer role](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to provided user or group - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [migration_admin_users](variables.tf#L15) | List of users authorized to create a new M4CE sources and perform all other migration operations, in IAM format. | list(string) | ✓ | | +| [migration_admin](variables.tf#L15) | User or group who can create a new M4CE sources and perform all other migration operations, in IAM format (`group:foo@example.com`). | string | ✓ | | | [migration_target_projects](variables.tf#L20) | List of target projects for m4ce workload migrations. | list(string) | ✓ | | -| [migration_viewer_users](variables.tf#L25) | List of users authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format. | list(string) | | [] | +| [migration_viewer](variables.tf#L25) | User or group authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format (`group:foo@example.com`). | string | | null | | [project_create](variables.tf#L31) | Parameters for the creation of the new project to host the M4CE backend. | object({…}) | | null | | [project_name](variables.tf#L40) | Name of an existing project or of the new project assigned as M4CE host project. | string | | "m4ce-host-project-000" | @@ -36,9 +35,7 @@ This sample creates\updates several distinct groups of resources: | name | description | sensitive | |---|---|:---:| | [m4ce_gmanaged_service_account](outputs.tf#L15) | Google managed service account created automatically during the migrate connector registration.. It is used by M4CE to perform activities on target projects. | | - - ## Test ```hcl @@ -48,8 +45,8 @@ module "test" { billing_account_id = "1234-ABCD-1234" parent = "folders/1234563" } - migration_admin_users = ["user:admin@example.com"] - migration_viewer_users = ["user:viewer@example.com"] + migration_admin = "user:admin@example.com" + migration_viewer = "user:viewer@example.com" migration_target_projects = [module.test-target-project.name] depends_on = [ module.test-target-project diff --git a/blueprints/cloud-operations/vm-migration/host-target-projects/main.tf b/blueprints/cloud-operations/vm-migration/host-target-projects/main.tf index a0e899d5..44dc354d 100644 --- a/blueprints/cloud-operations/vm-migration/host-target-projects/main.tf +++ b/blueprints/cloud-operations/vm-migration/host-target-projects/main.tf @@ -19,11 +19,11 @@ module "host-project" { : null ) name = var.project_name - parent = (var.project_create != null + parent = ( + var.project_create != null ? var.project_create.parent : null ) - services = [ "cloudresourcemanager.googleapis.com", "compute.googleapis.com", @@ -33,14 +33,24 @@ module "host-project" { "servicecontrol.googleapis.com", "vmmigration.googleapis.com", ] - project_create = var.project_create != null - - iam_additive = { - "roles/iam.serviceAccountKeyAdmin" = var.migration_admin_users, - "roles/iam.serviceAccountCreator" = var.migration_admin_users, - "roles/vmmigration.admin" = var.migration_admin_users, - "roles/vmmigration.viewer" = var.migration_viewer_users, + iam_bindings_additive = { + admin_sa_key_admin = { + role = "roles/iam.serviceAccountKeyAdmin" + member = var.migration_admin + } + admin_sa_creator = { + role = "roles/iam.serviceAccountCreator" + member = var.migration_admin + } + admin_vmm_admin = { + role = "roles/vmmigration.admin" + member = var.migration_admin + } + viewer_vmm_viewer = { + role = "roles/vmmigration.viewer" + member = var.migration_viewer + } } } @@ -56,7 +66,6 @@ module "target-projects" { source = "../../../../modules/project" name = each.key project_create = false - services = [ "servicemanagement.googleapis.com", "servicecontrol.googleapis.com", @@ -64,10 +73,18 @@ module "target-projects" { "cloudresourcemanager.googleapis.com", "compute.googleapis.com" ] - - iam_additive = { - "roles/resourcemanager.projectIamAdmin" = var.migration_admin_users, - "roles/compute.viewer" = var.migration_admin_users, - "roles/iam.serviceAccountUser" = var.migration_admin_users + iam_bindings_additive = { + admin_project_iam_admin = { + role = "roles/resourcemanager.projectIamAdmin" + member = var.migration_admin + } + admin_compute_viewer = { + role = "roles/compute.viewer" + member = var.migration_admin + } + admin_sa_user = { + role = "roles/iam.serviceAccountUser" + member = var.migration_admin + } } } diff --git a/blueprints/cloud-operations/vm-migration/host-target-projects/variables.tf b/blueprints/cloud-operations/vm-migration/host-target-projects/variables.tf index b4a91ad9..890fc823 100644 --- a/blueprints/cloud-operations/vm-migration/host-target-projects/variables.tf +++ b/blueprints/cloud-operations/vm-migration/host-target-projects/variables.tf @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -variable "migration_admin_users" { - description = "List of users authorized to create a new M4CE sources and perform all other migration operations, in IAM format." - type = list(string) +variable "migration_admin" { + description = "User or group who can create a new M4CE sources and perform all other migration operations, in IAM format (`group:foo@example.com`)." + type = string } variable "migration_target_projects" { @@ -22,10 +22,10 @@ variable "migration_target_projects" { type = list(string) } -variable "migration_viewer_users" { - description = "List of users authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format." - type = list(string) - default = [] +variable "migration_viewer" { + description = "User or group authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format (`group:foo@example.com`)." + type = string + default = null } variable "project_create" { diff --git a/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/README.md b/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/README.md index 5a1b3fbc..4504f29c 100644 --- a/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/README.md +++ b/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/README.md @@ -13,34 +13,33 @@ This is the high level diagram: This sample creates\update several distinct groups of resources: - projects - - M4CE host project with [required services](https://cloud.google.com/migrate/compute-engine/docs/5.0/how-to/enable-services#enabling_required_services_on_the_host_project) deployed on a new or existing project. - - M4CE target project prerequisites deployed on existing projects. + - M4CE host project with [required services](https://cloud.google.com/migrate/compute-engine/docs/5.0/how-to/enable-services#enabling_required_services_on_the_host_project) deployed on a new or existing project. + - M4CE target project prerequisites deployed on existing projects. - IAM - Create a [service account](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/migrate-connector#step-3) used at runtime by the M4CE connector for data replication - - Grant [migration admin roles](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to provided user accounts. - - Grant [migration viewer role](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to provided user accounts. - - Grant [roles on shared VPC](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/target-project#configure-permissions) to migration admins + - Grant [migration admin roles](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to provided user or group. + - Grant [migration viewer role](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to provided user or group. + - Grant [roles on shared VPC](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/target-project#configure-permissions) to migration user or group - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [migration_admin_users](variables.tf#L15) | List of users authorized to create a new M4CE sources and perform all other migration operations, in IAM format. | list(string) | ✓ | | +| [migration_admin](variables.tf#L15) | User or group who can create a new M4CE sources and perform all other migration operations, in IAM format (`group:foo@example.com`). | string | ✓ | | | [migration_target_projects](variables.tf#L20) | List of target projects for m4ce workload migrations. | list(string) | ✓ | | -| [sharedvpc_host_projects](variables.tf#L45) | List of host projects that share a VPC with the selected target projects. | list(string) | ✓ | | -| [migration_viewer_users](variables.tf#L25) | List of users authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format. | list(string) | | [] | -| [project_create](variables.tf#L30) | Parameters for the creation of the new project to host the M4CE backend. | object({…}) | | null | -| [project_name](variables.tf#L39) | Name of an existing project or of the new project assigned as M4CE host project. | string | | "m4ce-host-project-000" | +| [sharedvpc_host_projects](variables.tf#L46) | List of host projects that share a VPC with the selected target projects. | list(string) | ✓ | | +| [migration_viewer](variables.tf#L25) | User or group authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format (`group:foo@example.com`). | string | | null | +| [project_create](variables.tf#L31) | Parameters for the creation of the new project to host the M4CE backend. | object({…}) | | null | +| [project_name](variables.tf#L40) | Name of an existing project or of the new project assigned as M4CE host project. | string | | "m4ce-host-project-000" | ## Outputs | name | description | sensitive | |---|---|:---:| | [m4ce_gmanaged_service_account](outputs.tf#L15) | Google managed service account created automatically during the migrate connector registration. It is used by M4CE to perform activities on target projects. | | - ## Manual Steps + Once this blueprint is deployed the M4CE [m4ce_gmanaged_service_account](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/target-sa-compute-engine#configuring_the_default_service_account) has to be configured to grant the access to the shared VPC and allow the deploy of Compute Engine instances as the result of the migration. ## Test @@ -52,8 +51,8 @@ module "test" { billing_account_id = "1234-ABCD-1234" parent = "folders/1234563" } - migration_admin_users = ["user:admin@example.com"] - migration_viewer_users = ["user:viewer@example.com"] + migration_admin = "user:admin@example.com" + migration_viewer = "user:viewer@example.com" migration_target_projects = [module.test-target-project.name] sharedvpc_host_projects = [module.test-sharedvpc-host-project.name] depends_on = [ diff --git a/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/main.tf b/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/main.tf index 803d1dd2..ea09243b 100644 --- a/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/main.tf +++ b/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/main.tf @@ -23,7 +23,6 @@ module "host-project" { ? var.project_create.parent : null ) - services = [ "cloudresourcemanager.googleapis.com", "compute.googleapis.com", @@ -33,14 +32,24 @@ module "host-project" { "servicecontrol.googleapis.com", "vmmigration.googleapis.com", ] - project_create = var.project_create != null - - iam_additive = { - "roles/iam.serviceAccountKeyAdmin" = var.migration_admin_users, - "roles/iam.serviceAccountCreator" = var.migration_admin_users, - "roles/vmmigration.admin" = var.migration_admin_users, - "roles/vmmigration.viewer" = var.migration_viewer_users, + iam_bindings_additive = { + admin_sa_key_admin = { + role = "roles/iam.serviceAccountKeyAdmin" + member = var.migration_admin + } + admin_sa_creator = { + role = "roles/iam.serviceAccountCreator" + member = var.migration_admin + } + admin_vmm_admin = { + role = "roles/vmmigration.admin" + member = var.migration_admin + } + viewer_vmm_viewer = { + role = "roles/vmmigration.viewer" + member = var.migration_viewer + } } } @@ -51,12 +60,10 @@ module "m4ce-service-account" { } module "target-projects" { - for_each = toset(var.migration_target_projects) source = "../../../../modules/project" name = each.key project_create = false - services = [ "cloudresourcemanager.googleapis.com", "compute.googleapis.com", @@ -64,21 +71,27 @@ module "target-projects" { "servicemanagement.googleapis.com", "servicecontrol.googleapis.com", ] - - iam_additive = { - "roles/resourcemanager.projectIamAdmin" = var.migration_admin_users, - "roles/iam.serviceAccountUser" = var.migration_admin_users, + iam_bindings_additive = { + admin_project_iam_admin = { + role = "roles/resourcemanager.projectIamAdmin" + member = var.migration_admin + } + admin_sa_user = { + role = "roles/iam.serviceAccountUser" + member = var.migration_admin + } } } module "sharedvpc_host_project" { - for_each = toset(var.sharedvpc_host_projects) source = "../../../../modules/project" name = each.key project_create = false - - iam_additive = { - "roles/compute.viewer" = var.migration_admin_users, + iam_bindings_additive = { + admin_compute_viewer = { + role = "roles/compute.viewer" + member = var.migration_admin + } } } diff --git a/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/variables.tf b/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/variables.tf index 6b94ae88..556911e6 100644 --- a/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/variables.tf +++ b/blueprints/cloud-operations/vm-migration/host-target-sharedvpc/variables.tf @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -variable "migration_admin_users" { - description = "List of users authorized to create a new M4CE sources and perform all other migration operations, in IAM format." - type = list(string) +variable "migration_admin" { + description = "User or group who can create a new M4CE sources and perform all other migration operations, in IAM format (`group:foo@example.com`)." + type = string } variable "migration_target_projects" { @@ -22,11 +22,12 @@ variable "migration_target_projects" { type = list(string) } -variable "migration_viewer_users" { - description = "List of users authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format." - type = list(string) - default = [] +variable "migration_viewer" { + description = "User or group authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format (`group:foo@example.com`)." + type = string + default = null } + variable "project_create" { description = "Parameters for the creation of the new project to host the M4CE backend." type = object({ diff --git a/blueprints/cloud-operations/vm-migration/single-project/README.md b/blueprints/cloud-operations/vm-migration/single-project/README.md index bc51953d..9ce69477 100644 --- a/blueprints/cloud-operations/vm-migration/single-project/README.md +++ b/blueprints/cloud-operations/vm-migration/single-project/README.md @@ -13,21 +13,20 @@ This is the high level diagram: This sample creates several distinct groups of resources: - projects - - M4CE host project with [required services](https://cloud.google.com/migrate/compute-engine/docs/5.0/how-to/enable-services#enabling_required_services_on_the_host_project) deployed on a new or existing project. + - M4CE host project with [required services](https://cloud.google.com/migrate/compute-engine/docs/5.0/how-to/enable-services#enabling_required_services_on_the_host_project) deployed on a new or existing project. - networking - Default VPC network - IAM - One [service account](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/migrate-connector#step-3) used at runtime by the M4CE connector for data replication - - Grant [migration admin roles](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to admin user accounts - - Grant [migration viewer role](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to viewer user accounts + - Grant [migration admin roles](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to admin user or group + - Grant [migration viewer role](https://cloud.google.com/migrate/virtual-machines/docs/5.0/how-to/enable-services#using_predefined_roles) to viewer user or group - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [migration_admin_users](variables.tf#L15) | List of users authorized to create a new M4CE sources and perform all other migration operations, in IAM format. | list(string) | ✓ | | -| [migration_viewer_users](variables.tf#L20) | List of users authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format. | list(string) | | [] | +| [migration_admin](variables.tf#L15) | User or group who can create a new M4CE sources and perform all other migration operations, in IAM format (`group:foo@example.com`). | string | ✓ | | +| [migration_viewer](variables.tf#L20) | User or group authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format (`group:foo@example.com`). | string | | null | | [project_create](variables.tf#L26) | Parameters for the creation of the new project to host the M4CE backend. | object({…}) | | null | | [project_name](variables.tf#L35) | Name of an existing project or of the new project assigned as M4CE host an target project. | string | | "m4ce-host-project-000" | | [vpc_config](variables.tf#L41) | Parameters to create a simple VPC on the M4CE project. | object({…}) | | {…} | @@ -37,9 +36,7 @@ This sample creates several distinct groups of resources: | name | description | sensitive | |---|---|:---:| | [m4ce_gmanaged_service_account](outputs.tf#L15) | Google managed service account created automatically during the migrate connector registration. It is used by M4CE to perform activities on target projects. | | - - ## Test ```hcl @@ -49,8 +46,8 @@ module "test" { billing_account_id = "1234-ABCD-1234" parent = "folders/1234563" } - migration_admin_users = ["user:admin@example.com"] - migration_viewer_users = ["user:viewer@example.com"] + migration_admin = "user:admin@example.com" + migration_viewer = "user:viewer@example.com" } # tftest modules=5 resources=22 ``` diff --git a/blueprints/cloud-operations/vm-migration/single-project/main.tf b/blueprints/cloud-operations/vm-migration/single-project/main.tf index a6fee843..402752ff 100644 --- a/blueprints/cloud-operations/vm-migration/single-project/main.tf +++ b/blueprints/cloud-operations/vm-migration/single-project/main.tf @@ -23,7 +23,6 @@ module "landing-project" { ? var.project_create.parent : null ) - services = [ "cloudresourcemanager.googleapis.com", "compute.googleapis.com", @@ -34,14 +33,24 @@ module "landing-project" { "servicecontrol.googleapis.com", "vmmigration.googleapis.com" ] - project_create = var.project_create != null - - iam_additive = { - "roles/iam.serviceAccountKeyAdmin" = var.migration_admin_users, - "roles/iam.serviceAccountCreator" = var.migration_admin_users, - "roles/vmmigration.admin" = var.migration_admin_users, - "roles/vmmigration.viewer" = var.migration_viewer_users + iam_bindings_additive = { + admin_sa_key_admin = { + role = "roles/iam.serviceAccountKeyAdmin" + member = var.migration_admin + } + admin_sa_creator = { + role = "roles/iam.serviceAccountCreator" + member = var.migration_admin + } + admin_vmm_admin = { + role = "roles/vmmigration.admin" + member = var.migration_admin + } + viewer_vmm_viewer = { + role = "roles/vmmigration.viewer" + member = var.migration_viewer + } } } diff --git a/blueprints/cloud-operations/vm-migration/single-project/variables.tf b/blueprints/cloud-operations/vm-migration/single-project/variables.tf index 967a739a..eac89381 100644 --- a/blueprints/cloud-operations/vm-migration/single-project/variables.tf +++ b/blueprints/cloud-operations/vm-migration/single-project/variables.tf @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -variable "migration_admin_users" { - description = "List of users authorized to create a new M4CE sources and perform all other migration operations, in IAM format." - type = list(string) +variable "migration_admin" { + description = "User or group who can create a new M4CE sources and perform all other migration operations, in IAM format (`group:foo@example.com`)." + type = string } -variable "migration_viewer_users" { - description = "List of users authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format." - type = list(string) - default = [] +variable "migration_viewer" { + description = "User or group authorized to retrieve information about M4CE in the Google Cloud Console, in IAM format (`group:foo@example.com`)." + type = string + default = null } variable "project_create" { diff --git a/blueprints/cloud-operations/workload-identity-federation/README.md b/blueprints/cloud-operations/workload-identity-federation/README.md index ad6feaed..d9942c14 100644 --- a/blueprints/cloud-operations/workload-identity-federation/README.md +++ b/blueprints/cloud-operations/workload-identity-federation/README.md @@ -93,3 +93,19 @@ Once done testing, you can clean up resources by running `terraform destroy`. | [vm_public_ip_address](outputs.tf#L39) | Azure VM public IP address. | | + + diff --git a/blueprints/cloud-operations/workload-identity-federation/google-cloud.tf b/blueprints/cloud-operations/workload-identity-federation/google-cloud.tf index 372a2fcc..0924d08b 100644 --- a/blueprints/cloud-operations/workload-identity-federation/google-cloud.tf +++ b/blueprints/cloud-operations/workload-identity-federation/google-cloud.tf @@ -32,8 +32,11 @@ module "prj" { "sts.googleapis.com", ] project_create = var.project_create != null - iam_additive = { - "roles/viewer" : [module.sa.iam_email] + iam_bindings_additive = { + sa_viewer = { + member = module.sa.iam_email + role = "roles/viewer" + } } } diff --git a/blueprints/data-solutions/cloudsql-multiregion/README.md b/blueprints/data-solutions/cloudsql-multiregion/README.md index 1216c0aa..85f2594c 100644 --- a/blueprints/data-solutions/cloudsql-multiregion/README.md +++ b/blueprints/data-solutions/cloudsql-multiregion/README.md @@ -85,13 +85,14 @@ This implementation is intentionally minimal and easy to read. A real world use The example supports the configuration of a Shared VPC as an input variable. To deploy the solution on a Shared VPC, you have to configure the `network_config` variable: -``` +```hcl network_config = { - host_project = "PROJECT_ID" - network_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/global/networks/VPC_NAME" - subnet_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/$REGION/subnetworks/SUBNET_NAME" - cloudsql_psa_range = "10.60.0.0/24" - } + host_project = "PROJECT_ID" + network_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/global/networks/VPC_NAME" + subnet_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/$REGION/subnetworks/SUBNET_NAME" + cloudsql_psa_range = "10.60.0.0/24" +} +# tftest skip ``` To run this example, the Shared VPC project needs to have: @@ -137,7 +138,6 @@ terraform destroy The above command will delete the associated resources so there will be no billable charges made afterwards. - ## Variables | name | description | type | required | default | @@ -145,13 +145,14 @@ The above command will delete the associated resources so there will be no billa | [postgres_user_password](variables.tf#L40) | `postgres` user password. | string | ✓ | | | [prefix](variables.tf#L45) | Prefix used for resource names. | string | ✓ | | | [project_id](variables.tf#L63) | Project id, references existing project if `project_create` is null. | string | ✓ | | -| [data_eng_principals](variables.tf#L17) | Groups with Service Account Token creator role on service accounts in IAM format, only user supported on CloudSQL, eg 'user@domain.com'. | list(string) | | [] | +| [data_eng_principal](variables.tf#L17) | Group or user in IAM format (`group:foo@example.com`) with permissions to access resources and impersonate service accounts. | string | | null | | [network_config](variables.tf#L23) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | | [postgres_database](variables.tf#L34) | `postgres` database. | string | | "guestbook" | | [project_create](variables.tf#L54) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | | [regions](variables.tf#L68) | Map of instance_name => location where instances will be deployed. | map(string) | | {…} | | [service_encryption_keys](variables.tf#L81) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion configured. | map(string) | | null | | [sql_configuration](variables.tf#L87) | Cloud SQL configuration. | object({…}) | | {…} | +| [sql_users](variables.tf#L103) | Cloud SQL user emails. | list(string) | | [] | ## Outputs @@ -162,16 +163,14 @@ The above command will delete the associated resources so there will be no billa | [demo_commands](outputs.tf#L27) | Demo commands. | | | [ips](outputs.tf#L36) | IP address of each instance. | | | [project_id](outputs.tf#L41) | ID of the project containing all the instances. | | -| [service_accounts](outputs.tf#L46) | Service Accounts. | | - +| [service_account](outputs.tf#L46) | SQL client service Accounts. | | - ## Test ```hcl module "test" { source = "./fabric/blueprints/data-solutions/cloudsql-multiregion/" - data_eng_principals = ["dataeng@example.com"] + data_eng_principal = "group:dataeng@example.com" postgres_user_password = "my-root-password" project_id = "project" project_create = { @@ -180,5 +179,5 @@ module "test" { } prefix = "prefix" } -# tftest modules=10 resources=52 +# tftest modules=9 resources=43 ``` diff --git a/blueprints/data-solutions/cloudsql-multiregion/cloudsql.tf b/blueprints/data-solutions/cloudsql-multiregion/cloudsql.tf index 796cc1b6..52ff10e9 100644 --- a/blueprints/data-solutions/cloudsql-multiregion/cloudsql.tf +++ b/blueprints/data-solutions/cloudsql-multiregion/cloudsql.tf @@ -39,7 +39,7 @@ module "db" { } resource "google_sql_user" "users" { - for_each = toset(var.data_eng_principals) + for_each = toset(var.sql_users) project = module.project.project_id name = each.value instance = module.db.name @@ -47,8 +47,7 @@ resource "google_sql_user" "users" { } resource "google_sql_user" "service-account" { - for_each = toset(var.data_eng_principals) - project = module.project.project_id + project = module.project.project_id # Omit the .gserviceaccount.com suffix in the email name = regex("(.+)(.gserviceaccount)", module.service-account-sql.email)[0] instance = module.db.name diff --git a/blueprints/data-solutions/cloudsql-multiregion/datastorage.tf b/blueprints/data-solutions/cloudsql-multiregion/datastorage.tf deleted file mode 100644 index 1b45a97b..00000000 --- a/blueprints/data-solutions/cloudsql-multiregion/datastorage.tf +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "gcs" { - source = "../../../modules/gcs" - project_id = module.project.project_id - prefix = var.prefix - name = "data" - location = var.regions.primary - storage_class = "REGIONAL" - encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys[var.regions.primary], null) : null - force_destroy = true -} - -module "service-account-gcs" { - source = "../../../modules/iam-service-account" - project_id = module.project.project_id - name = "${var.prefix}-gcs" -} diff --git a/blueprints/data-solutions/cloudsql-multiregion/main.tf b/blueprints/data-solutions/cloudsql-multiregion/main.tf index 1fd76aa3..52f9175b 100644 --- a/blueprints/data-solutions/cloudsql-multiregion/main.tf +++ b/blueprints/data-solutions/cloudsql-multiregion/main.tf @@ -15,54 +15,26 @@ */ locals { - data_eng_principals_iam = [ - for k in var.data_eng_principals : - "user:${k}" - ] - - iam = { - # GCS roles - "roles/storage.objectAdmin" = [ - "serviceAccount:${module.project.service_accounts.robots.sql}", - module.service-account-gcs.iam_email, + iam_roles = { + data_eng = [ + "roles/owner" ] - # CloudSQL - "roles/cloudsql.admin" = local.data_eng_principals_iam - "roles/cloudsql.client" = concat( - local.data_eng_principals_iam, - [module.service-account-sql.iam_email] - ) - "roles/cloudsql.instanceUser" = concat( - local.data_eng_principals_iam, - [module.service-account-sql.iam_email] - ) - # compute engineering - "roles/compute.instanceAdmin.v1" = local.data_eng_principals_iam - "roles/compute.osLogin" = local.data_eng_principals_iam - "roles/compute.viewer" = local.data_eng_principals_iam - "roles/iap.tunnelResourceAccessor" = local.data_eng_principals_iam - # common roles - "roles/logging.admin" = local.data_eng_principals_iam - "roles/iam.serviceAccountUser" = concat( - local.data_eng_principals_iam - ) - "roles/iam.serviceAccountTokenCreator" = concat( - local.data_eng_principals_iam - ) - # network roles - "roles/compute.networkUser" = [ - "serviceAccount:${module.project.service_accounts.robots.sql}" + sql_robot = [ + "roles/compute.networkUser", + "roles/storage.objectAdmin" + ] + sql_sa = [ + "roles/cloudsql.client", + "roles/cloudsql.instanceUser" ] } - shared_vpc_project = try(var.network_config.host_project, null) - use_shared_vpc = var.network_config != null - subnet = ( local.use_shared_vpc ? var.network_config.subnet_self_link : values(module.vpc.0.subnet_self_links)[0] ) + use_shared_vpc = var.network_config != null vpc_self_link = ( local.use_shared_vpc ? var.network_config.network_self_link @@ -77,8 +49,26 @@ module "project" { billing_account = try(var.project_create.billing_account_id, null) project_create = var.project_create != null prefix = var.project_create == null ? null : var.prefix - iam = var.project_create != null ? local.iam : {} - iam_additive = var.project_create == null ? local.iam : {} + iam_bindings_additive = merge( + var.data_eng_principal == null ? {} : { + for r in local.iam_roles.data_eng : "data_eng-${r}" => { + member = var.data_eng_principal + role = r + } + }, + { + for r in local.iam_roles.sql_robot : "sql_robot-${r}" => { + member = "serviceAccount:${module.project.service_accounts.robots.sql}" + role = r + } + }, + { + for r in local.iam_roles.sql_sa : "sql_sa-${r}" => { + member = module.service-account-sql.iam_email + role = r + } + } + ) services = [ "cloudkms.googleapis.com", "compute.googleapis.com", @@ -92,12 +82,10 @@ module "project" { "storage.googleapis.com", "storage-component.googleapis.com", ] - shared_vpc_service_config = local.shared_vpc_project == null ? null : { attach = true host_project = local.shared_vpc_project } - service_encryption_key_ids = { compute = try(values(var.service_encryption_keys), []) sql = try(values(var.service_encryption_keys), []) @@ -120,7 +108,6 @@ module "vpc" { region = var.regions.primary } ] - psa_config = { ranges = { cloud-sql = var.sql_configuration.psa_range } routes = null @@ -145,3 +132,14 @@ module "nat" { name = "${var.prefix}-default" router_network = module.vpc.0.name } + +module "gcs" { + source = "../../../modules/gcs" + project_id = module.project.project_id + prefix = var.prefix + name = "data" + location = var.regions.primary + storage_class = "REGIONAL" + encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys[var.regions.primary], null) : null + force_destroy = true +} diff --git a/blueprints/data-solutions/cloudsql-multiregion/outputs.tf b/blueprints/data-solutions/cloudsql-multiregion/outputs.tf index 09960e13..44e3d786 100644 --- a/blueprints/data-solutions/cloudsql-multiregion/outputs.tf +++ b/blueprints/data-solutions/cloudsql-multiregion/outputs.tf @@ -43,10 +43,7 @@ output "project_id" { value = module.project.project_id } -output "service_accounts" { - description = "Service Accounts." - value = { - "gcs" = module.service-account-gcs.email - "sql" = module.service-account-sql.email - } +output "service_account" { + description = "SQL client service Accounts." + value = module.service-account-sql.email } diff --git a/blueprints/data-solutions/cloudsql-multiregion/variables.tf b/blueprints/data-solutions/cloudsql-multiregion/variables.tf index 65427792..6d6dfae5 100644 --- a/blueprints/data-solutions/cloudsql-multiregion/variables.tf +++ b/blueprints/data-solutions/cloudsql-multiregion/variables.tf @@ -14,10 +14,10 @@ * limitations under the License. */ -variable "data_eng_principals" { - description = "Groups with Service Account Token creator role on service accounts in IAM format, only user supported on CloudSQL, eg 'user@domain.com'." - type = list(string) - default = [] +variable "data_eng_principal" { + description = "Group or user in IAM format (`group:foo@example.com`) with permissions to access resources and impersonate service accounts." + type = string + default = null } variable "network_config" { @@ -99,3 +99,9 @@ variable "sql_configuration" { tier = "db-g1-small" } } + +variable "sql_users" { + description = "Cloud SQL user emails." + type = list(string) + default = [] +} diff --git a/blueprints/data-solutions/composer-2/README.md b/blueprints/data-solutions/composer-2/README.md index 3f665dbd..c43590e7 100644 --- a/blueprints/data-solutions/composer-2/README.md +++ b/blueprints/data-solutions/composer-2/README.md @@ -1,34 +1,39 @@ # Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key This blueprint creates a Private instance of [Cloud Composer version 2](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) on a VPC with a dedicated service account. Cloud Composer 2 is the new major version for Cloud Composer that supports: - - environment autoscaling - - workloads configuration: CPU, memory, and storage parameters for Airflow workers, schedulers, web server, and database. + +- environment autoscaling +- workloads configuration: CPU, memory, and storage parameters for Airflow workers, schedulers, web server, and database. Please consult the [documentation page](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) for an exhaustive comparison between Composer Version 1 and Version 2. The solution will use: - - Cloud Composer - - VPC with Private Service Access to deploy resources, if no Shared VPC configuration provided. - - Google Cloud NAT to access internet resources, if no Shared VPC configuration provided. + +- Cloud Composer +- VPC with Private Service Access to deploy resources, if no Shared VPC configuration provided. +- Google Cloud NAT to access internet resources, if no Shared VPC configuration provided. The solution supports as inputs: - - Shared VPC - - Cloud KMS CMEK keys + +- Shared VPC +- Cloud KMS CMEK keys This is the high level diagram: ![Cloud Composer 2 architecture overview](./diagram.png "Cloud Composer 2 architecture overview") -# Requirements +## Requirements + This blueprint will deploy all its resources into the project defined by the project_id variable. Please note that we assume this project already exists. However, if you provide the appropriate values to the `project_create` variable, the project will be created as part of the deployment. If `project_create` is left to null, the identity performing the deployment needs the owner role on the project defined by the `project_id` variable. Otherwise, the identity performing the deployment needs `resourcemanager.projectCreator` on the resource hierarchy node specified by `project_create.parent` and `billing.user` on the billing account specified by `project_create.billing_account_id`. -# Deployment +## Deployment + Run Terraform init: ```bash -$ terraform init +terraform init ``` Configure the Terraform variable in your terraform.tfvars file. You need to specify at least the following variables: @@ -41,23 +46,28 @@ prefix = "lc" You can run now: ```bash -$ terraform apply +terraform apply ``` You can now connect to your instance. -# Customizations +## Customizations + +### VPC -## VPC If a shared VPC is not configured, a VPC will be created within the project. The following IP ranges will be used: + - Cloudsql: `10.20.10.0/24` - GKE: `10.20.11.0/28` Change the code as needed to match your needed configuration, remember that these addresses should not overlap with any other range used in network. -## Shared VPC -As is often the case in real-world configurations, this blueprint accepts as input an existing [`Shared-VPC`](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. + +### Shared VPC + +As is often the case in real-world configurations, this blueprint accepts as input an existing [`Shared-VPC`](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Example: + ```tfvars network_config = { host_project = "PROJECT" @@ -68,42 +78,47 @@ network_config = { services = "services" } } +# tftest skip ``` Make sure that: + - The GKE API (`container.googleapis.com`) is enabled in the VPC host project. - The subnet has secondary ranges configured with 2 ranges: - - pods: `/22` example: `10.10.8.0/22` - - services = `/24` example: 10.10.12.0/24` + - pods: `/22` example: `10.10.8.0/22` + - services = `/24` example: 10.10.12.0/24` - Firewall rules are set, as described in the [documentation](https://cloud.google.com/composer/docs/composer-2/configure-private-ip#step_3_configure_firewall_rules) In order to run the example and deploy Cloud Composer on a shared VPC the identity running Terraform must have the following IAM role on the Shared VPC Host project. - - Compute Network Admin (roles/compute.networkAdmin) - - Compute Shared VPC Admin (roles/compute.xpnAdmin) + +- Compute Network Admin (roles/compute.networkAdmin) +- Compute Shared VPC Admin (roles/compute.xpnAdmin) ## Encryption -As is often the case in real-world configurations, this blueprint accepts as input an existing [`Cloud KMS keys`](https://cloud.google.com/kms/docs/cmek) via the `service_encryption_keys` variable. + +As is often the case in real-world configurations, this blueprint accepts as input an existing [`Cloud KMS keys`](https://cloud.google.com/kms/docs/cmek) via the `service_encryption_keys` variable. Example: + ```tfvars service_encryption_keys = { `europe/west1` = `projects/PROJECT/locations/REGION/keyRings/KR_NAME/cryptoKeys/KEY_NAME` } +# tftest skip ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L82) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L100) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [prefix](variables.tf#L83) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L101) | Project id, references existing project if `project_create` is null. | string | ✓ | | | [composer_config](variables.tf#L17) | Composer environment configuration. It accepts only following attributes: `environment_size`, `software_config` and `workloads_config`. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables. | object({…}) | | {…} | -| [iam_groups_map](variables.tf#L58) | Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}. | map(list(string)) | | null | -| [network_config](variables.tf#L64) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_create](variables.tf#L91) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [region](variables.tf#L105) | Reagion where instances will be deployed. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L111) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | +| [iam_bindings_additive](variables.tf#L58) | Map of Role => principal in IAM format (`group:foo@example.org`) to be added on the project. | map(list(string)) | | {} | +| [network_config](variables.tf#L65) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L92) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [region](variables.tf#L106) | Reagion where instances will be deployed. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L112) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | ## Outputs @@ -111,7 +126,6 @@ service_encryption_keys = { |---|---|:---:| | [composer_airflow_uri](outputs.tf#L17) | The URI of the Apache Airflow Web UI hosted within the Cloud Composer environment.. | | | [composer_dag_gcs](outputs.tf#L22) | The Cloud Storage prefix of the DAGs for the Cloud Composer environment. | | - ## Test diff --git a/blueprints/data-solutions/composer-2/main.tf b/blueprints/data-solutions/composer-2/main.tf index a9ee619c..3f2f9e84 100644 --- a/blueprints/data-solutions/composer-2/main.tf +++ b/blueprints/data-solutions/composer-2/main.tf @@ -15,14 +15,8 @@ */ locals { - iam = merge( - { - "roles/composer.worker" = [module.comp-sa.iam_email] - "roles/composer.ServiceAgentV2Ext" = ["serviceAccount:${module.project.service_accounts.robots.composer}"] - }, - var.iam_groups_map - ) - # Adding Roles on Service Identities Service account as per documentation: https://cloud.google.com/composer/docs/composer-2/configure-shared-vpc#edit_permissions_for_the_google_apis_service_account + # add Roles on Service Identities service account as per documentation + # https://cloud.google.com/composer/docs/composer-2/configure-shared-vpc#edit_permissions_for_the_google_apis_service_account _shared_vpc_bindings = { "roles/compute.networkUser" = [ "prj-cloudservices", "prj-robot-gke" @@ -34,11 +28,16 @@ locals { "prj-robot-gke" ] } - shared_vpc_role_members = { - prj-cloudservices = "serviceAccount:${module.project.service_accounts.cloud_services}" - prj-robot-gke = "serviceAccount:${module.project.service_accounts.robots.container-engine}" - prj-robot-cs = "serviceAccount:${module.project.service_accounts.robots.composer}" - } + orch_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_link + : values(module.vpc.0.subnet_self_links)[0] + ) + orch_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.vpc.0.self_link + ) # reassemble in a format suitable for for_each shared_vpc_bindings_map = { for binding in flatten([ @@ -47,27 +46,24 @@ locals { ] ]) : "${binding.role}-${binding.member}" => binding } - shared_vpc_project = try(var.network_config.host_project, null) - use_shared_vpc = var.network_config != null - + shared_vpc_role_members = { + prj-cloudservices = ( + "serviceAccount:${module.project.service_accounts.cloud_services}" + ) + prj-robot-gke = ( + "serviceAccount:${module.project.service_accounts.robots.container-engine}" + ) + prj-robot-cs = ( + "serviceAccount:${module.project.service_accounts.robots.composer}" + ) + } + use_shared_vpc = var.network_config != null vpc_self_link = ( local.use_shared_vpc ? var.network_config.network_self_link : module.vpc.0.self_link ) - - orch_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_link - : values(module.vpc.0.subnet_self_links)[0] - ) - - orch_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.vpc.0.self_link - ) } module "project" { @@ -77,8 +73,24 @@ module "project" { billing_account = try(var.project_create.billing_account_id, null) project_create = var.project_create != null prefix = var.project_create == null ? null : var.prefix - iam = var.project_create != null ? local.iam : {} - iam_additive = var.project_create == null ? local.iam : {} + iam_bindings_additive = merge( + { + composer_worker = { + member = module.comp-sa.iam_email + role = "roles/composer.worker" + }, + composer_service_agent = { + member = "serviceAccount:${module.project.service_accounts.robots.composer}" + role = "roles/composer.ServiceAgentV2Ext" + } + }, + { + for k, v in var.iam_bindings_additive : "${k}:${v}" => { + member = v + role = k + } + } + ) services = [ "artifactregistry.googleapis.com", "cloudkms.googleapis.com", @@ -94,19 +106,13 @@ module "project" { "storage.googleapis.com", "storage-component.googleapis.com", ] - shared_vpc_service_config = local.shared_vpc_project == null ? null : { attach = true host_project = local.shared_vpc_project } - service_encryption_key_ids = { composer = [try(lookup(var.service_encryption_keys, var.region, null), null)] } - - service_config = { - disable_on_destroy = false, disable_dependent_services = false - } } module "vpc" { diff --git a/blueprints/data-solutions/composer-2/variables.tf b/blueprints/data-solutions/composer-2/variables.tf index 77f095e3..c5d6ff89 100644 --- a/blueprints/data-solutions/composer-2/variables.tf +++ b/blueprints/data-solutions/composer-2/variables.tf @@ -55,10 +55,11 @@ variable "composer_config" { } } -variable "iam_groups_map" { - description = "Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}." +variable "iam_bindings_additive" { + description = "Map of Role => principal in IAM format (`group:foo@example.org`) to be added on the project." type = map(list(string)) - default = null + nullable = false + default = {} } variable "network_config" { diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index 46e9a130..3638dc08 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -15,21 +15,28 @@ # tfdoc:file:description drop off project and resources. locals { - iam_drp = { - "roles/bigquery.dataEditor" = [ - module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers + drp_iam = { + data_engineers = [ + "roles/bigquery.dataEditor", + "roles/bigquery.user" ] - "roles/bigquery.user" = [ - module.load-sa-df-0.iam_email, local.groups_iam.data-engineers + sa_drop_bq = [ + "roles/bigquery.dataEditor" ] - "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] - "roles/pubsub.subscriber" = [ - module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email + sa_drop_cs = [ + "roles/storage.objectCreator" ] - "roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email] - "roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email] - "roles/storage.objectAdmin" = [ - module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email + sa_drop_ps = [ + "roles/pubsub.publisher" + ] + sa_load = [ + "roles/bigquery.user", + "roles/pubsub.subscriber", + "roles/storage.objectAdmin" + ] + sa_orch = [ + "roles/pubsub.subscriber", + "roles/storage.objectViewer" ] } } @@ -39,10 +46,14 @@ module "drop-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" - iam = var.project_config.billing_account_id != null ? local.iam_drp : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null + prefix = local.use_projects ? null : var.prefix + name = ( + local.use_projects + ? var.project_config.project_ids.drop + : "${var.project_config.project_ids.drop}${local.project_suffix}" + ) + iam = local.use_projects ? {} : local.drp_iam_auth + iam_bindings_additive = !local.use_projects ? {} : local.drp_iam_additive services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -59,8 +70,6 @@ module "drop-project" { } } -# Cloud Storage - module "drop-sa-cs-0" { source = "../../../modules/iam-service-account" project_id = module.drop-project.project_id @@ -89,8 +98,6 @@ module "drop-cs-0" { # } } -# PubSub - module "drop-sa-ps-0" { source = "../../../modules/iam-service-account" project_id = module.drop-project.project_id @@ -111,8 +118,6 @@ module "drop-ps-0" { kms_key = try(local.service_encryption_keys.pubsub, null) } -# BigQuery - module "drop-sa-bq-0" { source = "../../../modules/iam-service-account" project_id = module.drop-project.project_id diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index 9702fce1..52fb5276 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -15,46 +15,38 @@ # tfdoc:file:description Load project and VPC. locals { - iam_load = { - "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] - "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, - module.load-sa-df-0.iam_email, - local.groups_iam.data-engineers + load_iam = { + data_engineers = [ + "roles/dataflow.admin" ] - "roles/dataflow.developer" = [ - local.groups_iam.data-engineers + robots_dataflow_load = [ + "roles/storage.objectAdmin" + ] + sa_load = [ + "roles/bigquery.jobUser", + "roles/dataflow.admin", + "roles/dataflow.worker", + "roles/storage.objectAdmin" + ] + sa_orch = [ + "roles/dataflow.admin" ] - "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] - "roles/storage.objectAdmin" = local.load_service_accounts } - load_service_accounts = [ - "serviceAccount:${module.load-project.service_accounts.robots.dataflow}", - module.load-sa-df-0.iam_email - ] - load_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_links.orchestration - : values(module.load-vpc.0.subnet_self_links)[0] - ) - load_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.load-vpc.0.self_link - ) } -# Project - module "load-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}" - iam = var.project_config.billing_account_id != null ? local.iam_load : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null + prefix = local.use_projects ? null : var.prefix + name = ( + local.use_projects + ? var.project_config.project_ids.load + : "${var.project_config.project_ids.load}${local.project_suffix}" + ) + iam = local.use_projects ? {} : local.load_iam_auth + iam_bindings_additive = !local.use_projects ? {} : local.load_iam_additive services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -106,8 +98,6 @@ module "load-cs-df-0" { encryption_key = try(local.service_encryption_keys.storage, null) } -# internal VPC resources - module "load-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index b3c19590..804250a5 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -15,58 +15,39 @@ # tfdoc:file:description Orchestration project and VPC. locals { - iam_orch = { - "roles/artifactregistry.admin" = [local.groups_iam.data-engineers] - "roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email] - "roles/bigquery.dataEditor" = [ - module.load-sa-df-0.iam_email, - module.transf-sa-df-0.iam_email, - local.groups_iam.data-engineers + orch_iam = { + data_engineers = [ + "roles/artifactregistry.admin", + "roles/bigquery.dataEditor", + "roles/bigquery.jobUser", + "roles/cloudbuild.builds.editor", + "roles/composer.environmentAndStorageObjectAdmin", + "roles/iam.serviceAccountUser", + "roles/iap.httpsResourceAccessor", + "roles/serviceusage.serviceUsageConsumer" ] - "roles/bigquery.jobUser" = [ - module.orch-sa-cmp-0.iam_email, - local.groups_iam.data-engineers + robots_cloudbuild = [ + "roles/storage.objectAdmin" ] - "roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers] - "roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email] - "roles/composer.admin" = [local.groups_iam.data-engineers] - "roles/composer.user" = [local.groups_iam.data-engineers] - "roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers] - "roles/composer.ServiceAgentV2Ext" = [ - "serviceAccount:${module.orch-project.service_accounts.robots.composer}" + robots_composer = [ + "roles/composer.ServiceAgentV2Ext", + "roles/storage.objectAdmin" ] - "roles/composer.worker" = [ - module.orch-sa-cmp-0.iam_email + sa_load = [ + "roles/artifactregistry.reader", + "roles/bigquery.dataEditor", + "roles/storage.objectViewer" ] - "roles/iam.serviceAccountUser" = [ - module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers + sa_orch = [ + "roles/bigquery.jobUser", + "roles/composer.worker", + "roles/iam.serviceAccountUser", + "roles/storage.objectAdmin" ] - "roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers] - "roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers] - "roles/storage.objectAdmin" = [ - module.orch-sa-cmp-0.iam_email, - module.orch-sa-df-build.iam_email, - "serviceAccount:${module.orch-project.service_accounts.robots.composer}", - "serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}", - local.groups_iam.data-engineers + sa_transf_df = [ + "roles/bigquery.dataEditor" ] - "roles/storage.objectViewer" = [module.load-sa-df-0.iam_email] } - orch_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_links.orchestration - : values(module.orch-vpc.0.subnet_self_links)[0] - ) - orch_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.orch-vpc.0.self_link - ) - - # Note: This formatting is needed for output purposes since the fabric artifact registry - # module doesn't yet expose the docker usage path of a registry folder in the needed format. - orch_docker_path = format("%s-docker.pkg.dev/%s/%s", - var.region, module.orch-project.project_id, module.orch-artifact-reg.name) } module "orch-project" { @@ -74,15 +55,17 @@ module "orch-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix + prefix = local.use_projects ? null : var.prefix name = ( - var.project_config.billing_account_id == null + local.use_projects ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" ) - iam = var.project_config.billing_account_id != null ? local.iam_orch : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null - oslogin = false + iam = local.use_projects ? {} : local.orch_iam_auth + iam_bindings_additive = !local.use_projects ? {} : local.orch_iam_additive + compute_metadata = { + enable-oslogin = "false" + } services = concat(var.project_services, [ "artifactregistry.googleapis.com", "bigquery.googleapis.com", @@ -112,8 +95,6 @@ module "orch-project" { } } -# Cloud Storage - module "orch-cs-0" { source = "../../../modules/gcs" project_id = module.orch-project.project_id @@ -124,8 +105,6 @@ module "orch-cs-0" { encryption_key = try(local.service_encryption_keys.storage, null) } -# internal VPC resources - module "orch-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index dd599c97..70c534b3 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -15,29 +15,25 @@ # tfdoc:file:description Transformation project and VPC. locals { - iam_trf = { - "roles/bigquery.jobUser" = [ - module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers + trf_iam = { + data_engineers = [ + "roles/bigquery.jobUser", + "roles/dataflow.admin" ] - "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers + robots_dataflow_trf = [ + "roles/storage.objectAdmin" ] - "roles/dataflow.worker" = [module.transf-sa-df-0.iam_email] - "roles/storage.objectAdmin" = [ - module.transf-sa-df-0.iam_email, - "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" + sa_orch = [ + "roles/dataflow.admin" + ] + sa_transf_bq = [ + "roles/bigquery.jobUser" + ] + sa_transf_df = [ + "roles/dataflow.worker", + "roles/storage.objectAdmin" ] } - transf_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_links.orchestration - : values(module.transf-vpc.0.subnet_self_links)[0] - ) - transf_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.transf-vpc.0.self_link - ) } module "transf-project" { @@ -45,10 +41,14 @@ module "transf-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" - iam = var.project_config.billing_account_id != null ? local.iam_trf : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null + prefix = local.use_projects ? null : var.prefix + name = ( + local.use_projects + ? var.project_config.project_ids.trf + : "${var.project_config.project_ids.trf}${local.project_suffix}" + ) + iam = local.use_projects ? {} : local.trf_iam_auth + iam_bindings_additive = !local.use_projects ? {} : local.trf_iam_additive services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -72,8 +72,6 @@ module "transf-project" { } } -# Cloud Storage - module "transf-sa-df-0" { source = "../../../modules/iam-service-account" project_id = module.transf-project.project_id @@ -101,8 +99,6 @@ module "transf-cs-df-0" { encryption_key = try(local.service_encryption_keys.storage, null) } -# BigQuery - module "transf-sa-bq-0" { source = "../../../modules/iam-service-account" project_id = module.transf-project.project_id @@ -120,8 +116,6 @@ module "transf-sa-bq-0" { } } -# internal VPC resources - module "transf-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index 67c43dae..8ba0f0f9 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -15,61 +15,48 @@ # tfdoc:file:description Data Warehouse projects. locals { - dwh_lnd_iam = { - "roles/bigquery.dataOwner" = [ - module.load-sa-df-0.iam_email, - ] - "roles/bigquery.dataViewer" = [ - module.transf-sa-df-0.iam_email, - module.transf-sa-bq-0.iam_email, - local.groups_iam.data-engineers - ] - "roles/bigquery.jobUser" = [ - module.load-sa-df-0.iam_email, local.groups_iam.data-engineers - ] - "roles/datacatalog.categoryAdmin" = [module.transf-sa-bq-0.iam_email] - "roles/datacatalog.tagTemplateViewer" = [local.groups_iam.data-engineers] - "roles/datacatalog.viewer" = [local.groups_iam.data-engineers] - "roles/storage.objectCreator" = [module.load-sa-df-0.iam_email] - "roles/storage.objectViewer" = [local.groups_iam.data-engineers] - } dwh_iam = { - "roles/bigquery.dataOwner" = [ - module.transf-sa-df-0.iam_email, - module.transf-sa-bq-0.iam_email, + data_analysts = [ + "roles/bigquery.dataViewer", + "roles/bigquery.jobUser", + "roles/datacatalog.viewer", + "roles/storage.objectViewer" ] - "roles/bigquery.dataViewer" = [ - local.groups_iam.data-analysts, - local.groups_iam.data-engineers + data_engineers = [ + "roles/bigquery.dataViewer", + "roles/bigquery.jobUser", + "roles/datacatalog.viewer", + "roles/storage.objectViewer" ] - "roles/bigquery.jobUser" = [ - module.transf-sa-bq-0.iam_email, - local.groups_iam.data-analysts, - local.groups_iam.data-engineers + sa_transf_bq = [ + "roles/bigquery.dataOwner", + "roles/bigquery.jobUser" ] - "roles/datacatalog.tagTemplateViewer" = [ - local.groups_iam.data-analysts, local.groups_iam.data-engineers + sa_transf_df = [ + "roles/bigquery.dataOwner", + "roles/storage.objectAdmin" + ] + } + lnd_iam = { + data_engineers = [ + "roles/bigquery.dataViewer", + "roles/bigquery.jobUser", + "roles/datacatalog.viewer", + "roles/storage.objectViewer" + ] + sa_load = [ + "roles/storage.objectCreator" + ] + sa_transf_bq = [ + "roles/bigquery.dataViewer", + "roles/datacatalog.categoryAdmin" + ] + sa_transf_df = [ + "roles/bigquery.dataOwner", + "roles/bigquery.dataViewer", + "roles/bigquery.jobUser" ] - "roles/datacatalog.viewer" = [ - local.groups_iam.data-analysts, local.groups_iam.data-engineers - ] - "roles/storage.objectViewer" = [ - local.groups_iam.data-analysts, local.groups_iam.data-engineers - ] - "roles/storage.objectAdmin" = [module.transf-sa-df-0.iam_email] } - dwh_services = concat(var.project_services, [ - "bigquery.googleapis.com", - "bigqueryreservation.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudkms.googleapis.com", - "compute.googleapis.com", - "dataflow.googleapis.com", - "pubsub.googleapis.com", - "servicenetworking.googleapis.com", - "storage.googleapis.com", - "storage-component.googleapis.com" - ]) } # Project @@ -79,11 +66,15 @@ module "dwh-lnd-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" - iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {} - iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {} - services = local.dwh_services + prefix = local.use_projects ? null : var.prefix + name = ( + local.use_projects + ? var.project_config.project_ids.dwh-lnd + : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" + ) + iam = local.use_projects ? {} : local.lnd_iam_auth + iam_bindings_additive = !local.use_projects ? {} : local.lnd_iam_additive + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -95,11 +86,15 @@ module "dwh-cur-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" - iam = var.project_config.billing_account_id != null ? local.dwh_iam : {} - iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {} - services = local.dwh_services + prefix = local.use_projects ? null : var.prefix + name = ( + local.use_projects + ? var.project_config.project_ids.dwh-cur + : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" + ) + iam = local.use_projects ? {} : local.dwh_iam_auth + iam_bindings_additive = !local.use_projects ? {} : local.dwh_iam_additive + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -111,19 +106,21 @@ module "dwh-conf-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" - iam = var.project_config.billing_account_id != null ? local.dwh_iam : null - iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null - services = local.dwh_services + prefix = local.use_projects ? null : var.prefix + name = ( + local.use_projects + ? var.project_config.project_ids.dwh-conf + : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" + ) + iam = local.use_projects ? {} : local.dwh_iam_auth + iam_bindings_additive = !local.use_projects ? {} : local.dwh_iam_additive + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] } } -# Bigquery - module "dwh-lnd-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.dwh-lnd-project.project_id @@ -148,8 +145,6 @@ module "dwh-conf-bq-0" { encryption_key = try(local.service_encryption_keys.bq, null) } -# Cloud storage - module "dwh-lnd-cs-0" { source = "../../../modules/gcs" project_id = module.dwh-lnd-project.project_id diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 0078b169..569c0a94 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -15,47 +15,56 @@ # tfdoc:file:description common project. locals { - iam_common = { - "roles/dlp.admin" = [local.groups_iam.data-security] - "roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers] - "roles/dlp.reader" = [local.groups_iam.data-engineers] - "roles/dlp.user" = [ - module.load-sa-df-0.iam_email, - module.transf-sa-df-0.iam_email, - local.groups_iam.data-engineers + cmn_iam = { + data_analysts = [ + # uncomment if access to all tagged columns is needed + # "roles/datacatalog.categoryFineGrainedReader", + "roles/datacatalog.viewer" ] - "roles/datacatalog.admin" = [local.groups_iam.data-security] - "roles/datacatalog.viewer" = [ - module.load-sa-df-0.iam_email, - module.transf-sa-df-0.iam_email, - module.transf-sa-bq-0.iam_email, - local.groups_iam.data-analysts + data_engineers = [ + "roles/dlp.estimatesAdmin", + "roles/dlp.reader", + "roles/dlp.user" ] - "roles/datacatalog.categoryFineGrainedReader" = [ - module.transf-sa-df-0.iam_email, - module.transf-sa-bq-0.iam_email, - # Uncomment if you want to grant access to `data-analyst` to all columns tagged. - # local.groups_iam.data-analysts + data_security = [ + "roles/datacatalog.admin", + "roles/dlp.admin" + ] + sa_load = [ + "roles/datacatalog.viewer", + "roles/dlp.user" + ] + sa_transf_bq = [ + "roles/datacatalog.categoryFineGrainedReader", + "roles/datacatalog.viewer" + ] + sa_transf_df = [ + "roles/datacatalog.categoryFineGrainedReader", + "roles/datacatalog.viewer", + "roles/dlp.user" ] } } + module "common-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" - iam = var.project_config.billing_account_id != null ? local.iam_common : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null + prefix = local.use_projects ? null : var.prefix + name = ( + local.use_projects + ? var.project_config.project_ids.common + : "${var.project_config.project_ids.common}${local.project_suffix}" + ) + iam = local.use_projects ? {} : local.cmn_iam_auth + iam_bindings_additive = !local.use_projects ? {} : local.cmn_iam_additive services = concat(var.project_services, [ "datacatalog.googleapis.com", "dlp.googleapis.com", ]) } -# Data Catalog Policy tag - module "common-datacatalog" { source = "../../../modules/data-catalog-policy-tag" project_id = module.common-project.project_id @@ -64,7 +73,8 @@ module "common-datacatalog" { tags = var.data_catalog_tags } -# To create KMS keys in the common project: uncomment this section and assigne key links accondingly in local.service_encryption_keys variable +# To create KMS keys in the common project: uncomment this section +# and assign key links accondingly in local.service_encryption_keys variable # module "cmn-kms-0" { # source = "../../../modules/kms" diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf index ea8fca09..b6d674b8 100644 --- a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf +++ b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf @@ -19,6 +19,10 @@ module "exp-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.exp : "${var.project_config.project_ids.exp}${local.project_suffix}" + prefix = local.use_projects ? null : var.prefix + name = ( + local.use_projects + ? var.project_config.project_ids.exp + : "${var.project_config.project_ids.exp}${local.project_suffix}" + ) } diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index b1b0ccbd..5c2cabee 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -202,8 +202,7 @@ project_config = { parent = "folders/1111111111" billing_account_id = "1111111-2222222-33333333" } -organization_domain = "domain.com" -~ +organization_domain = "domain.com" ``` For more fine details check variables on [`variables.tf`](./variables.tf) and update according to the desired configuration. Remember to create team groups described [below](#groups). @@ -229,8 +228,7 @@ module "data-platform" { } prefix = "myprefix" } - -# tftest modules=43 resources=285 +# tftest modules=43 resources=279 ``` ## Customizations @@ -262,19 +260,19 @@ You can find examples in the `[demo](./demo)` folder. | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [organization_domain](variables.tf#L159) | Organization domain. | string | ✓ | | -| [prefix](variables.tf#L164) | Prefix used for resource names. | string | ✓ | | -| [project_config](variables.tf#L173) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | -| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {…} | -| [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} | -| [data_force_destroy](variables.tf#L114) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [groups](variables.tf#L120) | User groups. | map(string) | | {…} | -| [location](variables.tf#L130) | Location used for multi-regional resources. | string | | "eu" | -| [network_config](variables.tf#L136) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_services](variables.tf#L207) | List of core services enabled on all projects. | list(string) | | […] | -| [project_suffix](variables.tf#L218) | Suffix used only for project ids. | string | | null | -| [region](variables.tf#L224) | Region used for regional resources. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L230) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | +| [organization_domain](variables.tf#L164) | Organization domain. | string | ✓ | | +| [prefix](variables.tf#L169) | Prefix used for resource names. | string | ✓ | | +| [project_config](variables.tf#L178) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {…} | +| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} | +| [data_force_destroy](variables.tf#L119) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | +| [groups](variables.tf#L125) | User groups. | map(string) | | {…} | +| [location](variables.tf#L135) | Location used for multi-regional resources. | string | | "eu" | +| [network_config](variables.tf#L141) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_services](variables.tf#L212) | List of core services enabled on all projects. | list(string) | | […] | +| [project_suffix](variables.tf#L223) | Suffix used only for project ids. | string | | null | +| [region](variables.tf#L229) | Region used for regional resources. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L235) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | ## Outputs @@ -296,18 +294,3 @@ Features to add in future releases: - Add example on how to use Cloud Data Loss Prevention - Add solution to handle Tables, Views, and Authorized Views lifecycle - Add solution to handle Metadata lifecycle - -## Test - -```hcl -module "test" { - source = "./fabric/blueprints/data-solutions/data-platform-foundations/" - organization_domain = "example.com" - project_config = { - billing_account_id = "123456-123456-123456" - parent = "folders/12345678" - } - prefix = "prefix" -} -# tftest modules=43 resources=285 -``` diff --git a/blueprints/data-solutions/data-platform-foundations/locals-01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/locals-01-dropoff.tf new file mode 100644 index 00000000..02b533d5 --- /dev/null +++ b/blueprints/data-solutions/data-platform-foundations/locals-01-dropoff.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _drp_iam = flatten([ + for principal, roles in local.drp_iam : [ + for role in roles : { + key = "${principal}-${role}" + principal = principal + role = role + } + ] + ]) + drp_iam_additive = { + for binding in local._drp_iam : binding.key => { + role = binding.role + member = local.iam_principals[binding.principal] + } + } + drp_iam_auth = { + for binding in local._drp_iam : + binding.role => local.iam_principals[binding.principal]... + } +} diff --git a/blueprints/data-solutions/data-platform-foundations/locals-02-load.tf b/blueprints/data-solutions/data-platform-foundations/locals-02-load.tf new file mode 100644 index 00000000..e395095c --- /dev/null +++ b/blueprints/data-solutions/data-platform-foundations/locals-02-load.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _load_iam = flatten([ + for principal, roles in local.load_iam : [ + for role in roles : { + key = "${principal}-${role}" + principal = principal + role = role + } + ] + ]) + load_iam_additive = { + for binding in local._load_iam : binding.key => { + role = binding.role + member = local.iam_principals[binding.principal] + } + } + load_iam_auth = { + for binding in local._load_iam : + binding.role => local.iam_principals[binding.principal]... + } + load_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.load-vpc.0.subnet_self_links)[0] + ) + load_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.load-vpc.0.self_link + ) +} diff --git a/blueprints/data-solutions/data-platform-foundations/locals-03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/locals-03-orchestration.tf new file mode 100644 index 00000000..741f32c2 --- /dev/null +++ b/blueprints/data-solutions/data-platform-foundations/locals-03-orchestration.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _orch_iam = flatten([ + for principal, roles in local.orch_iam : [ + for role in roles : { + key = "${principal}-${role}" + principal = principal + role = role + } + ] + ]) + orch_iam_additive = { + for binding in local._orch_iam : binding.key => { + role = binding.role + member = local.iam_principals[binding.principal] + } + } + orch_iam_auth = { + for binding in local._orch_iam : + binding.role => local.iam_principals[binding.principal]... + } + orch_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.orch-vpc.0.subnet_self_links)[0] + ) + orch_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.orch-vpc.0.self_link + ) + # TODO: use new artifact registry module output + orch_docker_path = format("%s-docker.pkg.dev/%s/%s", + var.region, module.orch-project.project_id, module.orch-artifact-reg.name) +} diff --git a/blueprints/data-solutions/data-platform-foundations/locals-04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/locals-04-transformation.tf new file mode 100644 index 00000000..8fd532bc --- /dev/null +++ b/blueprints/data-solutions/data-platform-foundations/locals-04-transformation.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _trf_iam = flatten([ + for principal, roles in local.trf_iam : [ + for role in roles : { + key = "${principal}-${role}" + principal = principal + role = role + } + ] + ]) + trf_iam_additive = { + for binding in local._trf_iam : binding.key => { + role = binding.role + member = local.iam_principals[binding.principal] + } + } + trf_iam_auth = { + for binding in local._trf_iam : + binding.role => local.iam_principals[binding.principal]... + } + transf_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.transf-vpc.0.subnet_self_links)[0] + ) + transf_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.transf-vpc.0.self_link + ) +} diff --git a/blueprints/data-solutions/data-platform-foundations/locals-05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/locals-05-datawarehouse.tf new file mode 100644 index 00000000..5bd652c8 --- /dev/null +++ b/blueprints/data-solutions/data-platform-foundations/locals-05-datawarehouse.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _dwh_iam = flatten([ + for principal, roles in local.dwh_iam : [ + for role in roles : { + key = "${principal}-${role}" + principal = principal + role = role + } + ] + ]) + _lnd_iam = flatten([ + for principal, roles in local.lnd_iam : [ + for role in roles : { + key = "${principal}-${role}" + principal = principal + role = role + } + ] + ]) + dwh_iam_additive = { + for binding in local._dwh_iam : binding.key => { + role = binding.role + member = local.iam_principals[binding.principal] + } + } + dwh_iam_auth = { + for binding in local._dwh_iam : + binding.role => local.iam_principals[binding.principal]... + } + dwh_services = concat(var.project_services, [ + "bigquery.googleapis.com", + "bigqueryreservation.googleapis.com", + "bigquerystorage.googleapis.com", + "cloudkms.googleapis.com", + "compute.googleapis.com", + "dataflow.googleapis.com", + "pubsub.googleapis.com", + "servicenetworking.googleapis.com", + "storage.googleapis.com", + "storage-component.googleapis.com" + ]) + lnd_iam_additive = { + for binding in local._lnd_iam : binding.key => { + role = binding.role + member = local.iam_principals[binding.principal] + } + } + lnd_iam_auth = { + for binding in local._lnd_iam : + binding.role => local.iam_principals[binding.principal]... + } +} diff --git a/blueprints/data-solutions/data-platform-foundations/locals-06-common.tf b/blueprints/data-solutions/data-platform-foundations/locals-06-common.tf new file mode 100644 index 00000000..4089af3d --- /dev/null +++ b/blueprints/data-solutions/data-platform-foundations/locals-06-common.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _cmn_iam = flatten([ + for principal, roles in local.cmn_iam : [ + for role in roles : { + key = "${principal}-${role}" + principal = principal + role = role + } + ] + ]) + cmn_iam_additive = { + for binding in local._cmn_iam : binding.key => { + role = binding.role + member = local.iam_principals[binding.principal] + } + } + cmn_iam_auth = { + for binding in local._cmn_iam : + binding.role => local.iam_principals[binding.principal]... + } +} diff --git a/blueprints/data-solutions/data-platform-foundations/main.tf b/blueprints/data-solutions/data-platform-foundations/main.tf index 536dc0b5..77944f4d 100644 --- a/blueprints/data-solutions/data-platform-foundations/main.tf +++ b/blueprints/data-solutions/data-platform-foundations/main.tf @@ -35,6 +35,22 @@ locals { groups_iam = { for k, v in local.groups : k => "group:${v}" } + iam_principals = { + data_analysts = "group:${local.groups.data-analysts}" + data_engineers = "group:${local.groups.data-engineers}" + data_security = "group:${local.groups.data-security}" + robots_cloudbuild = "serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}" + robots_composer = "serviceAccount:${module.orch-project.service_accounts.robots.composer}" + robots_dataflow_load = "serviceAccount:${module.load-project.service_accounts.robots.dataflow}" + robots_dataflow_trf = "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" + sa_drop_bq = module.drop-sa-bq-0.iam_email + sa_drop_cs = module.drop-sa-cs-0.iam_email + sa_drop_ps = module.drop-sa-ps-0.iam_email + sa_load = module.load-sa-df-0.iam_email + sa_orch = module.orch-sa-cmp-0.iam_email + sa_transf_bq = module.transf-sa-bq-0.iam_email, + sa_transf_df = module.transf-sa-df-0.iam_email, + } project_suffix = var.project_suffix == null ? "" : "-${var.project_suffix}" service_encryption_keys = var.service_encryption_keys shared_vpc_project = try(var.network_config.host_project, null) @@ -57,6 +73,7 @@ locals { ] ]) : "${binding.role}-${binding.member}" => binding } + use_projects = var.project_config.billing_account_id == null use_shared_vpc = var.network_config != null } diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index 9bb92e73..92a6316b 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -19,54 +19,59 @@ variable "composer_config" { type = object({ disable_deployment = optional(bool) environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") - software_config = optional(object({ - airflow_config_overrides = optional(any) - pypi_packages = optional(any) - env_variables = optional(map(string)) - image_version = string - }), { - image_version = "composer-2-airflow-2" - }) - workloads_config = optional(object({ - scheduler = optional(object( - { - cpu = number - memory_gb = number - storage_gb = number - count = number - } - ), { - cpu = 0.5 - memory_gb = 1.875 - storage_gb = 1 - count = 1 - }) - web_server = optional(object( - { - cpu = number - memory_gb = number - storage_gb = number - } - ), { - cpu = 0.5 - memory_gb = 1.875 - storage_gb = 1 - }) - worker = optional(object( - { - cpu = number - memory_gb = number - storage_gb = number - min_count = number - max_count = number - } - ), { - cpu = 0.5 - memory_gb = 1.875 - storage_gb = 1 - min_count = 1 - max_count = 3 - }) + software_config = optional( + object({ + airflow_config_overrides = optional(any) + pypi_packages = optional(any) + env_variables = optional(map(string)) + image_version = string + }), + { image_version = "composer-2-airflow-2" } + ) + workloads_config = optional( + object({ + scheduler = optional( + object({ + cpu = number + memory_gb = number + storage_gb = number + count = number + }), + { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 1 + } + ) + web_server = optional( + object({ + cpu = number + memory_gb = number + storage_gb = number + }), + { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + } + ) + worker = optional( + object({ + cpu = number + memory_gb = number + storage_gb = number + min_count = number + max_count = number + }), + { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + min_count = 1 + max_count = 3 + } + ) })) }) default = { diff --git a/blueprints/data-solutions/data-platform-minimal/01-landing.tf b/blueprints/data-solutions/data-platform-minimal/01-landing.tf index 10eb5974..94ecf5a3 100644 --- a/blueprints/data-solutions/data-platform-minimal/01-landing.tf +++ b/blueprints/data-solutions/data-platform-minimal/01-landing.tf @@ -16,9 +16,26 @@ locals { iam_lnd = { - "roles/storage.objectCreator" = [module.land-sa-0.iam_email] - "roles/storage.objectViewer" = [module.processing-sa-cmp-0.iam_email] - "roles/storage.objectAdmin" = [module.processing-sa-0.iam_email] + "roles/storage.objectCreator" = [ + module.land-sa-0.iam_email + ] + "roles/storage.objectViewer" = [ + module.processing-sa-cmp-0.iam_email + ] + "roles/storage.objectAdmin" = [ + module.processing-sa-0.iam_email + ] + } + # this only works because the service account module uses a static output + iam_lnd_additive = { + for k in flatten([ + for role, members in local.iam_lnd : [ + for member in members : { + role = role + member = member + } + ] + ]) : "${k.member}-${k.role}" => k } } @@ -27,14 +44,20 @@ module "land-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix + prefix = ( + var.project_config.billing_account_id == null ? null : var.prefix + ) name = ( var.project_config.billing_account_id == null ? var.project_config.project_ids.landing : "${var.project_config.project_ids.landing}${local.project_suffix}" ) - iam = var.project_config.billing_account_id != null ? local.iam_lnd : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_lnd : null + iam = ( + var.project_config.billing_account_id == null ? {} : local.iam_lnd + ) + iam_bindings_additive = ( + var.project_config.billing_account_id != null ? {} : local.iam_lnd_additive + ) services = [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-minimal/02-processing.tf b/blueprints/data-solutions/data-platform-minimal/02-processing.tf index 1d8cca2a..1bba98da 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-processing.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-processing.tf @@ -15,15 +15,23 @@ # tfdoc:file:description Processing project and VPC. locals { - iam_processing = { + iam_prc = { "roles/bigquery.jobUser" = [ module.processing-sa-cmp-0.iam_email, module.processing-sa-0.iam_email ] - "roles/composer.admin" = [local.groups_iam.data-engineers] - "roles/dataflow.admin" = [module.processing-sa-cmp-0.iam_email] - "roles/dataflow.worker" = [module.processing-sa-0.iam_email] - "roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers] + "roles/composer.admin" = [ + local.groups_iam.data-engineers + ] + "roles/dataflow.admin" = [ + module.processing-sa-cmp-0.iam_email + ] + "roles/dataflow.worker" = [ + module.processing-sa-0.iam_email + ] + "roles/composer.environmentAndStorageObjectAdmin" = [ + local.groups_iam.data-engineers + ] "roles/composer.ServiceAgentV2Ext" = [ "serviceAccount:${module.processing-project.service_accounts.robots.composer}" ] @@ -37,20 +45,39 @@ locals { module.processing-sa-0.iam_email ] "roles/iam.serviceAccountUser" = [ - module.processing-sa-cmp-0.iam_email, local.groups_iam.data-engineers + module.processing-sa-cmp-0.iam_email, + local.groups_iam.data-engineers + ] + "roles/iap.httpsResourceAccessor" = [ + local.groups_iam.data-engineers + ] + "roles/serviceusage.serviceUsageConsumer" = [ + local.groups_iam.data-engineers ] - "roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers] - "roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers] "roles/storage.admin" = [ module.processing-sa-cmp-0.iam_email, "serviceAccount:${module.processing-project.service_accounts.robots.composer}", local.groups_iam.data-engineers ] } + # this only works because the service account module uses a static output + iam_prc_additive = { + for k in flatten([ + for role, members in local.iam_prc : [ + for member in members : { + role = role + member = member + } + ] + ]) : "${k.member}-${k.role}" => k + } processing_subnet = ( local.use_shared_vpc ? var.network_config.subnet_self_link - : try(module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], null) + : try( + module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], + null + ) ) processing_vpc = ( local.use_shared_vpc @@ -64,15 +91,23 @@ module "processing-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix + prefix = ( + var.project_config.billing_account_id == null ? null : var.prefix + ) name = ( var.project_config.billing_account_id == null ? var.project_config.project_ids.processing : "${var.project_config.project_ids.processing}${local.project_suffix}" ) - iam = var.project_config.billing_account_id != null ? local.iam_processing : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_processing : null - oslogin = false + iam = ( + var.project_config.billing_account_id == null ? {} : local.iam_prc + ) + iam_bindings_additive = ( + var.project_config.billing_account_id != null ? {} : local.iam_prc_additive + ) + compute_metadata = { + enable-oslogin = "false" + } services = [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-minimal/03-curated.tf b/blueprints/data-solutions/data-platform-minimal/03-curated.tf index 4b72f761..8bff815f 100644 --- a/blueprints/data-solutions/data-platform-minimal/03-curated.tf +++ b/blueprints/data-solutions/data-platform-minimal/03-curated.tf @@ -15,15 +15,32 @@ # tfdoc:file:description Data curated project and resources. locals { - cur_iam = { - "roles/bigquery.dataOwner" = [module.processing-sa-0.iam_email] + cur_services = [ + "bigquery.googleapis.com", + "bigqueryreservation.googleapis.com", + "bigquerystorage.googleapis.com", + "cloudkms.googleapis.com", + "cloudresourcemanager.googleapis.com", + "compute.googleapis.com", + "iam.googleapis.com", + "servicenetworking.googleapis.com", + "serviceusage.googleapis.com", + "stackdriver.googleapis.com", + "storage.googleapis.com", + "storage-component.googleapis.com" + ] + iam_cur = { + "roles/bigquery.dataOwner" = [ + module.processing-sa-0.iam_email + ] "roles/bigquery.dataViewer" = [ module.cur-sa-0.iam_email, local.groups_iam.data-analysts, local.groups_iam.data-engineers ] "roles/bigquery.jobUser" = [ - module.processing-sa-0.iam_email, # Remove once bug is fixed. https://github.com/apache/airflow/issues/32106 + # Remove once bug is fixed. https://github.com/apache/airflow/issues/32106 + module.processing-sa-0.iam_email, module.cur-sa-0.iam_email, local.groups_iam.data-analysts, local.groups_iam.data-engineers @@ -43,22 +60,21 @@ locals { local.groups_iam.data-analysts, local.groups_iam.data-engineers ] - "roles/storage.objectAdmin" = [module.processing-sa-0.iam_email] + "roles/storage.objectAdmin" = [ + module.processing-sa-0.iam_email + ] + } + # this only works because the service account module uses a static output + iam_cur_additive = { + for k in flatten([ + for role, members in local.iam_cur : [ + for member in members : { + role = role + member = member + } + ] + ]) : "${k.member}-${k.role}" => k } - cur_services = [ - "bigquery.googleapis.com", - "bigqueryreservation.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudkms.googleapis.com", - "cloudresourcemanager.googleapis.com", - "compute.googleapis.com", - "iam.googleapis.com", - "servicenetworking.googleapis.com", - "serviceusage.googleapis.com", - "stackdriver.googleapis.com", - "storage.googleapis.com", - "storage-component.googleapis.com" - ] } # Project @@ -68,15 +84,21 @@ module "cur-project" { parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix + prefix = ( + var.project_config.billing_account_id == null ? null : var.prefix + ) name = ( var.project_config.billing_account_id == null ? var.project_config.project_ids.curated : "${var.project_config.project_ids.curated}${local.project_suffix}" ) - iam = var.project_config.billing_account_id != null ? local.cur_iam : {} - iam_additive = var.project_config.billing_account_id == null ? local.cur_iam : {} - services = local.cur_services + iam = ( + var.project_config.billing_account_id != null ? {} : local.iam_cur + ) + iam_bindings_additive = ( + var.project_config.billing_account_id == null ? {} : local.iam_cur_additive + ) + services = local.cur_services service_encryption_key_ids = { bq = [var.service_encryption_keys.bq] storage = [var.service_encryption_keys.storage] diff --git a/blueprints/data-solutions/data-platform-minimal/04-common.tf b/blueprints/data-solutions/data-platform-minimal/04-common.tf index 52f6e84f..5080a083 100644 --- a/blueprints/data-solutions/data-platform-minimal/04-common.tf +++ b/blueprints/data-solutions/data-platform-minimal/04-common.tf @@ -15,15 +15,23 @@ # tfdoc:file:description Common project and resources. locals { - iam_common = { - "roles/dlp.admin" = [local.groups_iam.data-security] - "roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers] - "roles/dlp.reader" = [local.groups_iam.data-engineers] + iam_cmn = { + "roles/dlp.admin" = [ + local.groups_iam.data-security + ] + "roles/dlp.estimatesAdmin" = [ + local.groups_iam.data-engineers + ] + "roles/dlp.reader" = [ + local.groups_iam.data-engineers + ] "roles/dlp.user" = [ module.processing-sa-0.iam_email, local.groups_iam.data-engineers ] - "roles/datacatalog.admin" = [local.groups_iam.data-security] + "roles/datacatalog.admin" = [ + local.groups_iam.data-security + ] "roles/datacatalog.viewer" = [ module.processing-sa-0.iam_email, local.groups_iam.data-analysts @@ -32,20 +40,37 @@ locals { module.processing-sa-0.iam_email ] } + # this only works because the service account module uses a static output + iam_cmn_additive = { + for k in flatten([ + for role, members in local.iam_cmn : [ + for member in members : { + role = role + member = member + } + ] + ]) : "${k.member}-${k.role}" => k + } } module "common-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix + prefix = ( + var.project_config.billing_account_id == null ? null : var.prefix + ) name = ( var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" ) - iam = var.project_config.billing_account_id != null ? local.iam_common : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null + iam = ( + var.project_config.billing_account_id == null ? {} : local.iam_cmn + ) + iam_bindings_additive = ( + var.project_config.billing_account_id != null ? {} : local.iam_cmn_additive + ) services = [ "cloudresourcemanager.googleapis.com", "datacatalog.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md index 32622abe..1f4eb777 100644 --- a/blueprints/data-solutions/data-platform-minimal/README.md +++ b/blueprints/data-solutions/data-platform-minimal/README.md @@ -12,6 +12,32 @@ The following diagram is a high-level reference of the resources created and man A set of demo [Airflow pipelines](./demo/) are also part of this blueprint: they can be run on top of the foundational infrastructure to verify and test the setup. + +- [Design overview and choices](#design-overview-and-choices) +- [Project structure](#project-structure) +- [Roles](#roles) +- [Service accounts](#service-accounts) +- [User groups](#user-groups) + - [Virtual Private Cloud (VPC) design](#virtual-private-cloud-vpc-design) + - [IP ranges and subnetting](#ip-ranges-and-subnetting) + - [Resource naming conventions](#resource-naming-conventions) + - [Encryption](#encryption) +- [Data Anonymization](#data-anonymization) +- [Data Catalog](#data-catalog) +- [How to run this script](#how-to-run-this-script) +- [Variable configuration](#variable-configuration) +- [How to use this blueprint from Terraform](#how-to-use-this-blueprint-from-terraform) +- [Customizations](#customizations) + - [Assign roles at BQ Dataset level](#assign-roles-at-bq-dataset-level) + - [Project Configuration](#project-configuration) + - [Shared VPC](#shared-vpc) + - [Customer Managed Encryption key](#customer-managed-encryption-key) +- [Demo pipeline](#demo-pipeline) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + ## Design overview and choices Despite its simplicity, this stage implements the basics of a design that we've seen working well for various customers. @@ -203,7 +229,7 @@ module "data-platform" { prefix = "myprefix" } -# tftest modules=23 resources=123 +# tftest modules=23 resources=135 ``` ## Customizations diff --git a/blueprints/data-solutions/gcs-to-bq-with-least-privileges/main.tf b/blueprints/data-solutions/gcs-to-bq-with-least-privileges/main.tf index a096592f..33faf353 100644 --- a/blueprints/data-solutions/gcs-to-bq-with-least-privileges/main.tf +++ b/blueprints/data-solutions/gcs-to-bq-with-least-privileges/main.tf @@ -14,16 +14,6 @@ locals { iam = { - "roles/iam.serviceAccountUser" = [ - module.service-account-orch.iam_email - ] - "roles/iam.serviceAccountTokenCreator" = var.data_eng_principals - # GCS roles - "roles/storage.objectAdmin" = [ - module.service-account-df.iam_email, - module.service-account-landing.iam_email - ] - # BigQuery roles "roles/bigquery.admin" = var.data_eng_principals "roles/bigquery.dataOwner" = [ module.service-account-df.iam_email @@ -34,9 +24,7 @@ locals { "roles/bigquery.jobUser" = [ module.service-account-bq.iam_email ] - # Compute "roles/compute.viewer" = var.data_eng_principals - # Dataflow roles "roles/dataflow.admin" = concat( [module.service-account-orch.iam_email], var.data_eng_principals @@ -45,6 +33,25 @@ locals { "roles/dataflow.worker" = [ module.service-account-df.iam_email, ] + "roles/iam.serviceAccountUser" = [ + module.service-account-orch.iam_email + ] + "roles/iam.serviceAccountTokenCreator" = var.data_eng_principals + "roles/storage.objectAdmin" = [ + module.service-account-df.iam_email, + module.service-account-landing.iam_email + ] + } + # this only works because the service account module uses a static output + iam_additive = { + for k in flatten([ + for role, members in local.iam : [ + for member in members : { + role = role + member = member + } + ] + ]) : "${k.member}-${k.role}" => k } network_subnet_selflink = try( module.vpc[0].subnets["${var.region}/subnet"].self_link, @@ -75,8 +82,12 @@ module "project" { "storage.googleapis.com", "storage-component.googleapis.com", ] - iam = var.project_config.billing_account_id != null ? local.iam : {} - iam_additive = var.project_config.billing_account_id == null ? local.iam : {} + iam = ( + var.project_config.billing_account_id != null ? local.iam : {} + ) + iam_bindings_additive = ( + var.project_config.billing_account_id == null ? local.iam_additive : {} + ) shared_vpc_service_config = var.network_config.host_project == null ? null : { attach = true host_project = var.network_config.host_project diff --git a/blueprints/data-solutions/shielded-folder/README.md b/blueprints/data-solutions/shielded-folder/README.md index 83e9589d..ed177d27 100644 --- a/blueprints/data-solutions/shielded-folder/README.md +++ b/blueprints/data-solutions/shielded-folder/README.md @@ -153,25 +153,24 @@ terraform init terraform apply ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [access_policy_config](variables.tf#L17) | Provide 'access_policy_create' values if a folder scoped Access Policy creation is needed, uses existing 'policy_name' otherwise. Parent is in 'organizations/123456' format. Policy will be created scoped to the folder. | object({…}) | ✓ | | | [folder_config](variables.tf#L49) | Provide 'folder_create' values if folder creation is needed, uses existing 'folder_id' otherwise. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | -| [organization](variables.tf#L128) | Organization details. | object({…}) | ✓ | | -| [prefix](variables.tf#L136) | Prefix used for resources that need unique names. | string | ✓ | | -| [project_config](variables.tf#L141) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | +| [organization](variables.tf#L129) | Organization details. | object({…}) | ✓ | | +| [prefix](variables.tf#L137) | Prefix used for resources that need unique names. | string | ✓ | | +| [project_config](variables.tf#L142) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | [data_dir](variables.tf#L29) | Relative path for the folder storing configuration data. | string | | "data" | | [enable_features](variables.tf#L35) | Flag to enable features on the solution. | object({…}) | | {…} | | [groups](variables.tf#L65) | User groups. | object({…}) | | {} | -| [kms_keys](variables.tf#L75) | KMS keys to create, keyed by name. | map(object({…})) | | {} | -| [log_locations](variables.tf#L86) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | -| [log_sinks](variables.tf#L103) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | -| [vpc_sc_access_levels](variables.tf#L161) | VPC SC access level definitions. | map(object({…})) | | {} | -| [vpc_sc_egress_policies](variables.tf#L190) | VPC SC egress policy definitions. | map(object({…})) | | {} | -| [vpc_sc_ingress_policies](variables.tf#L210) | VPC SC ingress policy definitions. | map(object({…})) | | {} | +| [kms_keys](variables.tf#L75) | KMS keys to create, keyed by name. | map(object({…})) | | {} | +| [log_locations](variables.tf#L87) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | +| [log_sinks](variables.tf#L104) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | +| [vpc_sc_access_levels](variables.tf#L162) | VPC SC access level definitions. | map(object({…})) | | {} | +| [vpc_sc_egress_policies](variables.tf#L191) | VPC SC egress policy definitions. | map(object({…})) | | {} | +| [vpc_sc_ingress_policies](variables.tf#L211) | VPC SC ingress policy definitions. | map(object({…})) | | {} | ## Outputs @@ -180,7 +179,6 @@ terraform apply | [folders](outputs.tf#L15) | Folders id. | | | [folders_sink_writer_identities](outputs.tf#L23) | Folders id. | | | [kms_keys](outputs.tf#L31) | Cloud KMS encryption keys created. | | - ## Test diff --git a/blueprints/data-solutions/shielded-folder/kms.tf b/blueprints/data-solutions/shielded-folder/kms.tf index 708b5cda..9953d458 100644 --- a/blueprints/data-solutions/shielded-folder/kms.tf +++ b/blueprints/data-solutions/shielded-folder/kms.tf @@ -25,12 +25,9 @@ locals { for k, v in var.kms_keys : k => v if contains(v.locations, loc) } } - kms_log_locations = distinct(flatten([ for k, v in local.kms_log_sink_keys : compact(v.locations) ])) - - # Log sink keys kms_log_sink_keys = { "storage" = { labels = {} @@ -61,8 +58,12 @@ module "sec-project" { name = var.project_config.project_ids["sec-core"] parent = module.folder.id billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null && var.enable_features.encryption - prefix = var.project_config.billing_account_id == null ? null : var.prefix + project_create = ( + var.project_config.billing_account_id != null && var.enable_features.encryption + ) + prefix = ( + var.project_config.billing_account_id == null ? null : var.prefix + ) group_iam = { (local.groups.workload-security) = [ "roles/editor" @@ -76,17 +77,23 @@ module "sec-project" { } module "sec-kms" { - for_each = var.enable_features.encryption ? toset(local.kms_locations) : toset([]) + for_each = ( + var.enable_features.encryption + ? toset(local.kms_locations) + : toset([]) + ) source = "../../../modules/kms" project_id = module.sec-project[0].project_id keyring = { location = each.key name = "sec-${each.key}" } - # rename to `key_iam` to switch to authoritative bindings - key_iam_additive = { + key_iam = { for k, v in local.kms_locations_keys[each.key] : k => v.iam } + key_iam_bindings_additive = { + for k, v in local.kms_locations_keys[each.key] : k => v.iam_bindings_additive + } keys = local.kms_locations_keys[each.key] } diff --git a/blueprints/data-solutions/shielded-folder/variables.tf b/blueprints/data-solutions/shielded-folder/variables.tf index f4ec8acb..5bb80d57 100644 --- a/blueprints/data-solutions/shielded-folder/variables.tf +++ b/blueprints/data-solutions/shielded-folder/variables.tf @@ -75,10 +75,11 @@ variable "groups" { variable "kms_keys" { description = "KMS keys to create, keyed by name." type = map(object({ - iam = optional(map(list(string)), {}) - labels = optional(map(string), {}) - locations = optional(list(string), ["global", "europe", "europe-west1"]) - rotation_period = optional(string, "7776000s") + iam = optional(map(list(string)), {}) + iam_bindings_additive = optional(map(map(any)), {}) + labels = optional(map(string), {}) + locations = optional(list(string), ["global", "europe", "europe-west1"]) + rotation_period = optional(string, "7776000s") })) default = {} } diff --git a/blueprints/data-solutions/sqlserver-alwayson/main.tf b/blueprints/data-solutions/sqlserver-alwayson/main.tf index 4a255015..c9297003 100644 --- a/blueprints/data-solutions/sqlserver-alwayson/main.tf +++ b/blueprints/data-solutions/sqlserver-alwayson/main.tf @@ -73,9 +73,6 @@ module "project" { "compute.googleapis.com", "secretmanager.googleapis.com", ] - - iam = {} - iam_additive = {} shared_vpc_service_config = var.shared_vpc_project_id == null ? null : { attach = true host_project = var.shared_vpc_project_id diff --git a/blueprints/factories/project-factory/README.md b/blueprints/factories/project-factory/README.md index 1025c825..d144a11a 100644 --- a/blueprints/factories/project-factory/README.md +++ b/blueprints/factories/project-factory/README.md @@ -1,270 +1,95 @@ -# Minimal Project Factory +# Project Factory -This module implements a minimal, opinionated project factory (see [Factories](../README.md) for rationale) that allows for the creation of projects. +This is a working example of how to manage project creation at scale, by wrapping the [project module](../../../modules/project/) and driving it via external data, either directly provided or parsed via YAML files. -While the module can be invoked by manually populating the required variables, its interface is meant for the massive creation of resources leveraging a set of well-defined YaML documents, as shown in the examples below. +The wrapping layer around the project module is intentionally thin, so that -The Project Factory is meant to be executed by a Service Account (or a regular user) having this minimal set of permissions over your resources: +- all the features of the project module are available +- no "magic" or hidden side effects are implemented in code +- debugging and integration of new features is simple -* **Org level** - a custom role for networking operations including the following permissions - * `"compute.organizations.enableXpnResource"`, - * `"compute.organizations.disableXpnResource"`, - * `"compute.subnetworks.setIamPolicy"`, - * `"dns.networks.bindPrivateDNSZone"` - * and role `"roles/orgpolicy.policyAdmin"` -* **on each folder** where projects will be created - * `"roles/logging.admin"` - * `"roles/owner"` - * `"roles/resourcemanager.folderAdmin"` - * `"roles/resourcemanager.projectCreator"` -* **on the host project** for the Shared VPC/s - * `"roles/browser"` - * `"roles/compute.viewer"` - * `"roles/dns.admin"` +The code is meant to be executed by a high level service accounts with powerful permissions: + +- Shared VPC connection if service project attachment is desired +- project creation on the nodes (folder or org) where projects will be defined + +The module also supports optional creation of specific resources that usually part of the project creation flow: + +- service accounts used for VM instances, and associated basic roles +- KMS key encrypt/decrypt permissions for service identities in the project +- membership in VPC SC standard or bridge perimeters + +Compared to the previous version of this code, network-related resources (DNS zones, VPC subnets, etc.) have been removed as they are not typically in scope for the team who manages project creation, and adding them when needed requires just a few trivial code changes. ## Example -### Directory structure - -``` -. -├── data -│ ├── defaults.yaml -│ └── projects -│ ├── project-example-one.yaml -│ ├── project-example-two.yaml -│ └── project-example-three.yaml -├── main.tf -└── terraform.tfvars - -``` - -### Terraform code - ```hcl -locals { - defaults = yamldecode(file(local._defaults_file)) - projects = { - for f in fileset("${local._data_dir}", "**/*.yaml") : - trimsuffix(f, ".yaml") => yamldecode(file("${local._data_dir}/${f}")) +module "project-factory" { + source = "./fabric/blueprints/factories/project-factory" + data_defaults = { + billing_account = "012345-67890A-ABCDEF" + } + data_merges = { + labels = { + environment = "test" + } + services = [ + "stackdriver.googleapis.com" + ] + } + data_overrides = { + contacts = { + "admin@example.com" = ["ALL"] + } + prefix = "test-pf" + } + factory_data = { + data_path = "data" } - # these are usually set via variables - _base_dir = "./fabric/blueprints/factories/project-factory" - _data_dir = "${local._base_dir}/sample-data/projects/" - _defaults_file = "${local._base_dir}/sample-data/defaults.yaml" } - -module "projects" { - source = "./fabric/blueprints/factories/project-factory" - for_each = local.projects - defaults = local.defaults - project_id = each.key - descriptive_name = try(each.value.descriptive_name, null) - billing_account_id = try(each.value.billing_account_id, null) - billing_alert = try(each.value.billing_alert, null) - dns_zones = try(each.value.dns_zones, []) - essential_contacts = try(each.value.essential_contacts, []) - folder_id = each.value.folder_id - group_iam = try(each.value.group_iam, {}) - iam = try(each.value.iam, {}) - kms_service_agents = try(each.value.kms_service_agents, {}) - labels = try(each.value.labels, {}) - org_policies = try(each.value.org_policies, {}) - prefix = each.value.prefix - service_accounts = try(each.value.service_accounts, {}) - services = try(each.value.services, []) - service_identities_iam = try(each.value.service_identities_iam, {}) - vpc = try(each.value.vpc, null) -} -# tftest modules=7 resources=38 inventory=example.yaml -``` - -### Projects configuration - -```yaml -# ./data/defaults.yaml -# The following applies as overridable defaults for all projects -# All attributes are required - -billing_account_id: 012345-67890A-BCDEF0 -billing_alert: - amount: 1000 - thresholds: - current: [0.5, 0.8] - forecasted: [0.5, 0.8] - credit_treatment: INCLUDE_ALL_CREDITS -environment_dns_zone: prod.gcp.example.com -essential_contacts: [] -labels: - environment: production - department: legal - application: my-legal-bot -notification_channels: [] -shared_vpc_self_link: https://www.googleapis.com/compute/v1/projects/project-example-host-project/global/networks/vpc-one -vpc_host_project: project-example-host-project - +# tftest modules=6 resources=12 files=prj-app-1,prj-app-2 inventory=example.yaml ``` ```yaml -# ./data/projects/project-example-one.yaml -# One file per project - projects will be named after the filename - -# [opt] Billing account id - overrides default if set -billing_account_id: 012345-67890A-BCDEF0 - -# [opt] Billing alerts config - overrides default if set -billing_alert: - amount: 10 - thresholds: - current: - - 0.5 - - 0.8 - forecasted: [] - -# [opt] DNS zones to be created as children of the environment_dns_zone defined in defaults -dns_zones: - - lorem - - ipsum - -# [opt] Contacts for billing alerts and important notifications -essential_contacts: - - team-a-contacts@example.com - -# Folder the project will be created as children of -folder_id: folders/012345678901 - -# [opt] Authoritative IAM bindings in group => [roles] format -group_iam: - test-team-foobar@fast-lab-0.gcp-pso-italy.net: - - roles/compute.admin - -# [opt] Authoritative IAM bindings in role => [principals] format -# Generally used to grant roles to service accounts external to the project -iam: - roles/compute.admin: - - serviceAccount:service-account - -# [opt] Service robots and keys they will be assigned as cryptoKeyEncrypterDecrypter -# in service => [keys] format -kms_service_agents: - compute: [key1, key2] - storage: [key1, key2] - -# [opt] Labels for the project - merged with the ones defined in defaults +billing_account: 012345-67890A-BCDEF0 labels: - environment: prod - -# [opt] Org policy overrides defined at project level -org_policies: - compute.disableGuestAttributesAccess: - rules: - - enforce: true - compute.trustedImageProjects: - rules: - - allow: - values: - - projects/fast-dev-iac-core-0 - compute.vmExternalIpAccess: - rules: - - deny: - all: true - -# [opt] Service account to create for the project and their roles on the project -# in name => [roles] format -service_accounts: - another-service-account: - - roles/compute.admin - my-service-account: - - roles/compute.admin - -# [opt] IAM bindings on the service account resources. -# in name => {role => [members]} format -service_accounts_iam: - another-service-account: - - roles/iam.serviceAccountTokenCreator: - - group: app-team-1@example.com - -# [opt] APIs to enable on the project. + app: app-1 + team: foo +service_encryption_key_ids: + compute: + - projects/kms-central-prj/locations/europe-west3/keyRings/my-keyring/cryptoKeys/europe3-gce services: - - storage.googleapis.com - - stackdriver.googleapis.com - - compute.googleapis.com +- storage.googleapis.com +service_accounts: + app-1-be: {} + app-1-fe: {} -# [opt] Roles to assign to the robots service accounts in robot => [roles] format -services_iam: - compute: - - roles/storage.objectViewer +# tftest-file id=prj-app-1 path=data/prj-app-1.yaml +``` - # [opt] VPC setup. - # If set enables the `compute.googleapis.com` service and configures - # service project attachment -vpc: +```yaml +labels: + app: app-1 + team: foo +service_accounts: + app-2-be: {} - # [opt] If set, enables the container API - gke_setup: - - # Grants "roles/container.hostServiceAgentUser" to the container robot if set - enable_host_service_agent: false - - # Grants "roles/compute.securityAdmin" to the container robot if set - enable_security_admin: true - - # Host project the project will be service project of - host_project: fast-prod-net-spoke-0 - - # [opt] Services for which set up the IAM in the host project - service_iam_grants: - - dataproc.googleapis.com - - # [opt] Roles to rant service project service identities in host project - service_identity_iam: - "roles/compute.networkUser": - - cloudservices - - container-engine - - # [opt] Subnets in the host project where principals will be granted networkUser - # in region/subnet-name => [principals] - subnets_iam: - europe-west1/prod-default-ew1: - - user:foobar@example.com - - serviceAccount:service-account1@my-project.iam.gserviceaccount.com +# tftest-file id=prj-app-2 path=data/prj-app-2.yaml ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | -| [prefix](variables.tf#L144) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L153) | Project id. | string | ✓ | | -| [billing_alert](variables.tf#L22) | Billing alert configuration. | object({…}) | | null | -| [defaults](variables.tf#L35) | Project factory default values. | object({…}) | | null | -| [descriptive_name](variables.tf#L57) | Name of the project name. Used for project name instead of `name` variable. | string | | null | -| [dns_zones](variables.tf#L63) | DNS private zones to create as child of var.defaults.environment_dns_zone. | list(string) | | [] | -| [essential_contacts](variables.tf#L69) | Email contacts to be used for billing and GCP notifications. | list(string) | | [] | -| [folder_id](variables.tf#L75) | Folder ID for the folder where the project will be created. | string | | null | -| [group_iam](variables.tf#L81) | Custom IAM settings in group => [role] format. | map(list(string)) | | {} | -| [group_iam_additive](variables.tf#L87) | Custom additive IAM settings in group => [role] format. | map(list(string)) | | {} | -| [iam](variables.tf#L93) | Custom IAM settings in role => [principal] format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L99) | Custom additive IAM settings in role => [principal] format. | map(list(string)) | | {} | -| [kms_service_agents](variables.tf#L105) | KMS IAM configuration in as service => [key]. | map(list(string)) | | {} | -| [labels](variables.tf#L111) | Labels to be assigned at project level. | map(string) | | {} | -| [org_policies](variables.tf#L117) | Org-policy overrides at project level. | map(object({…})) | | {} | -| [service_accounts](variables.tf#L158) | Service accounts to be created, and roles assigned them on the project. | map(list(string)) | | {} | -| [service_accounts_additive](variables.tf#L164) | Service accounts to be created, and roles assigned them on the project additively. | map(list(string)) | | {} | -| [service_accounts_iam](variables.tf#L170) | IAM bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string))) | | {} | -| [service_accounts_iam_additive](variables.tf#L177) | IAM additive bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string))) | | {} | -| [service_identities_iam](variables.tf#L184) | Custom IAM settings for service identities in service => [role] format. | map(list(string)) | | {} | -| [service_identities_iam_additive](variables.tf#L191) | Custom additive IAM settings for service identities in service => [role] format. | map(list(string)) | | {} | -| [services](variables.tf#L198) | Services to be enabled for the project. | list(string) | | [] | -| [vpc](variables.tf#L205) | VPC configuration for the project. | object({…}) | | {…} | +| [factory_data](variables.tf#L83) | Project data from either YAML files or externally parsed data. | object({…}) | ✓ | | +| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | object({…}) | | {} | +| [data_merges](variables.tf#L44) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | +| [data_overrides](variables.tf#L63) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | ## Outputs | name | description | sensitive | |---|---|:---:| -| [project](outputs.tf#L19) | The project resource as return by the `project` module. | | -| [project_id](outputs.tf#L29) | Project ID. | | - +| [projects](outputs.tf#L17) | Project module outputs. | | +| [service_accounts](outputs.tf#L22) | Service account emails. | | diff --git a/blueprints/factories/project-factory/factory.tf b/blueprints/factories/project-factory/factory.tf new file mode 100644 index 00000000..dac843df --- /dev/null +++ b/blueprints/factories/project-factory/factory.tf @@ -0,0 +1,100 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _data = ( + var.factory_data.data != null + ? var.factory_data.data + : { + for f in fileset("${local._data_path}", "**/*.yaml") : + trimsuffix(f, ".yaml") => yamldecode(file("${local._data_path}/${f}")) + } + ) + _data_path = var.factory_data.data_path == null ? null : pathexpand( + var.factory_data.data_path + ) + projects = { + for k, v in local._data : k => merge(v, { + billing_account = coalesce( + var.data_overrides.billing_account, + try(v.billing_account, null), + var.data_defaults.billing_account + ) + contacts = coalesce( + var.data_overrides.contacts, + try(v.contacts, null), + var.data_defaults.contacts + ) + labels = coalesce( + try(v.labels, null), + var.data_defaults.labels + ) + metric_scopes = coalesce( + try(v.metric_scopes, null), + var.data_defaults.metric_scopes + ) + prefix = coalesce( + var.data_overrides.prefix, + try(v.prefix, null), + var.data_defaults.prefix + ) + service_encryption_key_ids = coalesce( + var.data_overrides.service_encryption_key_ids, + try(v.service_encryption_key_ids, null), + var.data_defaults.service_encryption_key_ids + ) + service_perimeter_bridges = coalesce( + var.data_overrides.service_perimeter_bridges, + try(v.service_perimeter_bridges, null), + var.data_defaults.service_perimeter_bridges + ) + service_perimeter_standard = try(coalesce( + var.data_overrides.service_perimeter_standard, + try(v.service_perimeter_standard, null), + var.data_defaults.service_perimeter_standard + ), null) + services = coalesce( + var.data_overrides.services, + try(v.services, null), + var.data_defaults.services + ) + shared_vpc_service_config = coalesce( + try(v.shared_vpc_service_config, null), + var.data_defaults.shared_vpc_service_config + ) + tag_bindings = coalesce( + var.data_overrides.tag_bindings, + try(v.tag_bindings, null), + var.data_defaults.tag_bindings + ) + # non-project resources + service_accounts = coalesce( + var.data_overrides.service_accounts, + try(v.service_accounts, null), + var.data_defaults.service_accounts + ) + }) + } + service_accounts = flatten([ + for k, v in local.projects : [ + for name, opts in v.service_accounts : { + project = k + name = name + options = opts + } + ] + ]) +} diff --git a/blueprints/factories/project-factory/main.tf b/blueprints/factories/project-factory/main.tf index 9dbecf2f..7d173a11 100644 --- a/blueprints/factories/project-factory/main.tf +++ b/blueprints/factories/project-factory/main.tf @@ -14,222 +14,68 @@ * limitations under the License. */ -locals { - _gke_config_service_identity_iam = { - "roles/compute.networkUser" = compact([ - var.vpc.gke_setup.enable_host_service_agent ? "container-engine" : null, - local.vpc_cloudservices ? "cloudservices" : null - ]) - "roles/compute.securityAdmin" = compact([ - var.vpc.gke_setup.enable_security_admin ? "container-engine" : null, - ]) - "roles/container.hostServiceAgentUser" = compact([ - var.vpc.gke_setup.enable_host_service_agent ? "container-engine" : null - ]) - } - - _group_iam = { - for r in local._group_iam_bindings : r => [ - for k, v in var.group_iam : - "group:${k}" if try(index(v, r), null) != null - ] - } - _group_iam_additive = { - for r in local._group_iam_additive_bindings : r => [ - for k, v in var.group_iam_additive : - "group:${k}" if try(index(v, r), null) != null - ] - } - _group_iam_bindings = distinct(flatten(values(var.group_iam))) - _group_iam_additive_bindings = distinct(flatten(values(var.group_iam_additive))) - - _service_accounts_iam = { - for r in local._service_accounts_iam_bindings : r => [ - for k, v in var.service_accounts : - module.service-accounts[k].iam_email - if try(index(v, r), null) != null - ] - } - _service_accounts_iam_bindings = distinct(flatten( - values(var.service_accounts) +module "projects" { + source = "../../../modules/project" + for_each = local.projects + billing_account = each.value.billing_account + name = each.key + parent = try(each.value.parent, null) + prefix = each.value.prefix + auto_create_network = try(each.value.auto_create_network, false) + compute_metadata = try(each.value.compute_metadata, {}) + # TODO: concat lists for each key + contacts = merge( + each.value.contacts, var.data_merges.contacts + ) + default_service_account = try(each.value.default_service_account, "keep") + descriptive_name = try(each.value.descriptive_name, null) + group_iam = try(each.value.group_iam, {}) + iam = try(each.value.iam, {}) + iam_bindings = try(each.value.iam_bindings, {}) + iam_bindings_additive = try(each.value.iam_bindings_additive, {}) + labels = each.value.labels + lien_reason = try(each.value.lien_reason, null) + logging_data_access = try(each.value.logging_data_access, {}) + logging_exclusions = try(each.value.logging_exclusions, {}) + logging_sinks = try(each.value.logging_sinks, {}) + metric_scopes = distinct(concat( + each.value.metric_scopes, var.data_merges.metric_scopes )) - _service_accounts_iam_additive = { - for r in local._service_accounts_iam_additive_bindings : r => [ - for k, v in var.service_accounts_additive : - module.service-accounts[k].iam_email - if try(index(v, r), null) != null - ] - } - _service_accounts_iam_additive_bindings = distinct(flatten( - values(var.service_accounts_additive) + service_encryption_key_ids = merge( + each.value.service_encryption_key_ids, + var.data_merges.service_encryption_key_ids + ) + service_perimeter_bridges = distinct(concat( + each.value.service_perimeter_bridges, + var.data_merges.service_perimeter_bridges )) - _services = concat([ - "billingbudgets.googleapis.com", - "essentialcontacts.googleapis.com", - "orgpolicy.googleapis.com", - ], - length(var.dns_zones) > 0 ? ["dns.googleapis.com"] : [], - try(var.vpc.gke_setup, null) != null ? ["container.googleapis.com"] : [], - var.vpc != null ? ["compute.googleapis.com"] : [], + service_perimeter_standard = each.value.service_perimeter_standard + services = distinct(concat( + each.value.services, + var.data_merges.services + )) + shared_vpc_service_config = each.value.shared_vpc_service_config + tag_bindings = merge( + each.value.tag_bindings, + var.data_merges.tag_bindings ) - _service_identities_roles = distinct(flatten(values(var.service_identities_iam))) - _service_identities_iam = { - for role in local._service_identities_roles : role => [ - for service, roles in var.service_identities_iam : - "serviceAccount:${module.project.service_accounts.robots[service]}" - if contains(roles, role) - ] - } - _service_identities_roles_additive = distinct(flatten(values(var.service_identities_iam_additive))) - _service_identities_iam_additive = { - for role in local._service_identities_roles_additive : role => [ - for service, roles in var.service_identities_iam_additive : - "serviceAccount:${module.project.service_accounts.robots[service]}" - if contains(roles, role) - ] - } - _vpc_subnet_bindings = ( - var.vpc.subnets_iam == null || var.vpc.host_project == null - ? [] - : flatten([ - for subnet, members in var.vpc.subnets_iam : [ - for member in members : { - region = split("/", subnet)[0] - subnet = split("/", subnet)[1] - member = member - } - ] - ]) - ) - billing_account_id = coalesce( - var.billing_account_id, try(var.defaults.billing_account_id, "") - ) - billing_alert = ( - var.billing_alert == null - ? try(var.defaults.billing_alert, null) - : var.billing_alert - ) - essential_contacts = concat( - try(var.defaults.essential_contacts, []), var.essential_contacts - ) - iam = { - for role in distinct(concat( - keys(var.iam), - keys(local._group_iam), - keys(local._service_accounts_iam), - keys(local._service_identities_iam), - )) : - role => concat( - try(var.iam[role], []), - try(local._group_iam[role], []), - try(local._service_accounts_iam[role], []), - try(local._service_identities_iam[role], []), - ) - } - iam_additive = { - for role in distinct(concat( - keys(var.iam_additive), - keys(local._group_iam_additive), - keys(local._service_accounts_iam_additive), - keys(local._service_identities_iam_additive), - )) : - role => concat( - try(var.iam_additive[role], []), - try(local._group_iam_additive[role], []), - try(local._service_accounts_iam_additive[role], []), - try(local._service_identities_iam_additive[role], []), - ) - } - labels = merge( - coalesce(var.labels, {}), coalesce(try(var.defaults.labels, {}), {}) - ) - services = distinct(concat(var.services, local._services)) - vpc_cloudservices = ( - var.vpc.gke_setup.enable_host_service_agent || - contains(var.services, "compute.googleapis.com") - ) - - vpc_service_identity_iam = { - for role in setunion(keys(local._gke_config_service_identity_iam), keys(var.vpc.service_identity_iam)) : - role => setunion( - lookup(local._gke_config_service_identity_iam, role, []), - lookup(var.vpc.service_identity_iam, role, []), - ) - } - vpc_subnet_bindings = { - for binding in local._vpc_subnet_bindings : - "${binding.subnet}:${binding.member}" => binding - } -} - -module "billing-alert" { - for_each = local.billing_alert == null ? {} : { 1 = 1 } - source = "../../../modules/billing-budget" - billing_account = local.billing_account_id - name = "${module.project.project_id} budget" - amount = local.billing_alert.amount - thresholds = local.billing_alert.thresholds - credit_treatment = local.billing_alert.credit_treatment - notification_channels = var.defaults.notification_channels - projects = ["projects/${module.project.number}"] - email_recipients = { - project_id = module.project.project_id - emails = local.essential_contacts - } -} - -module "dns" { - source = "../../../modules/dns" - for_each = toset(var.dns_zones) - project_id = coalesce(var.vpc.host_project, module.project.project_id) - name = each.value - zone_config = { - domain = "${each.value}.${var.defaults.environment_dns_zone}" - private = { - client_networks = [var.defaults.shared_vpc_self_link] - } - } -} - -module "project" { - source = "../../../modules/project" - billing_account = local.billing_account_id - name = var.project_id - descriptive_name = var.descriptive_name - prefix = var.prefix - contacts = { for c in local.essential_contacts : c => ["ALL"] } - iam = local.iam - iam_additive = local.iam_additive - labels = local.labels - org_policies = try(var.org_policies, {}) - parent = var.folder_id - service_encryption_key_ids = var.kms_service_agents - services = local.services - shared_vpc_service_config = var.vpc == null ? null : { - host_project = var.vpc.host_project - # these are non-authoritative - service_identity_iam = local.vpc_service_identity_iam - service_iam_grants = var.vpc.service_iam_grants - } } module "service-accounts" { - source = "../../../modules/iam-service-account" - for_each = var.service_accounts - name = each.key - project_id = module.project.project_id - iam = lookup(var.service_accounts_iam, each.key, null) -} - -resource "google_compute_subnetwork_iam_member" "default" { - for_each = local.vpc_subnet_bindings - project = var.vpc.host_project - subnetwork = "projects/${var.vpc.host_project}/regions/${each.value.region}/subnetworks/${each.value.subnet}" - region = each.value.region - role = "roles/compute.networkUser" - member = ( - lookup(var.service_accounts, each.value.member, null) != null - ? module.service-accounts[each.value.member].iam_email - : each.value.member + source = "../../../modules/iam-service-account" + for_each = { + for k in local.service_accounts : "${k.project}-${k.name}" => k + } + name = each.value.name + project_id = module.projects[each.value.project].project_id + iam_project_roles = ( + try(each.value.options.default_roles, null) == null + ? {} + : { + (module.projects[each.value.project].project_id) = [ + "roles/logging.logWriter", + "roles/monitoring.metricWriter" + ] + } ) } diff --git a/blueprints/factories/project-factory/outputs.tf b/blueprints/factories/project-factory/outputs.tf index a989eaba..99653a15 100644 --- a/blueprints/factories/project-factory/outputs.tf +++ b/blueprints/factories/project-factory/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,23 +14,15 @@ * limitations under the License. */ -# TODO(): proper outputs - -output "project" { - description = "The project resource as return by the `project` module." - value = module.project - - depends_on = [ - google_compute_subnetwork_iam_member.default, - module.dns - ] +output "projects" { + description = "Project module outputs." + value = module.projects } -output "project_id" { - description = "Project ID." - value = module.project.project_id - depends_on = [ - google_compute_subnetwork_iam_member.default, - module.dns - ] +output "service_accounts" { + description = "Service account emails." + # TODO: group by project + value = { + for k, v in module.service-accounts : k => v.email + } } diff --git a/blueprints/factories/project-factory/sample-data/defaults.yaml b/blueprints/factories/project-factory/sample-data/defaults.yaml deleted file mode 100644 index 72ed3f0d..00000000 --- a/blueprints/factories/project-factory/sample-data/defaults.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# skip boilerplate check - -billing_account_id: 012345-67890A-BCDEF0 - -# [opt] Setup for billing alerts -billing_alert: - amount: 1000 - thresholds: - current: [0.5, 0.8] - forecasted: [0.5, 0.8] - credit_treatment: INCLUDE_ALL_CREDITS - -environment_dns_zone: dev.example.org - -# [opt] Contacts for billing alerts and important notifications -essential_contacts: ["team-contacts@example.com"] - -# [opt] Labels set for all projects -labels: - environment: dev - department: accounting - application: example-app - foo: bar - -# [opt] Additional notification channels for billing -notification_channels: [] -shared_vpc_self_link: projects/foo/networks/bar -prefix: test -vpc_host_project: diff --git a/blueprints/factories/project-factory/sample-data/projects/project.yaml b/blueprints/factories/project-factory/sample-data/projects/project.yaml deleted file mode 100644 index e5957c77..00000000 --- a/blueprints/factories/project-factory/sample-data/projects/project.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# skip boilerplate check - -# [opt] Billing account id - overrides default if set -billing_account_id: 012345-67890A-BCDEF0 - -# [opt] Billing alerts config - overrides default if set -billing_alert: - amount: 10 - thresholds: - current: - - 0.5 - - 0.8 - forecasted: [] - credit_treatment: INCLUDE_ALL_CREDITS - -# [opt] DNS zones to be created as children of the environment_dns_zone defined in defaults -dns_zones: - - lorem - - ipsum - -# [opt] Contacts for billing alerts and important notifications -essential_contacts: - - team-a-contacts@example.com - -# Folder the project will be created as children of -folder_id: folders/012345678901 - -# [opt] Authoritative IAM bindings in group => [roles] format -group_iam: - test-team-foobar@fast-lab-0.gcp-pso-italy.net: - - roles/compute.admin - -# [opt] Authoritative IAM bindings in role => [principals] format -# Generally used to grant roles to service accounts external to the project -iam: - roles/compute.admin: - - serviceAccount:service-account - -# [opt] Service robots and keys they will be assigned as cryptoKeyEncrypterDecrypter -# in service => [keys] format -kms_service_agents: - compute: [key1, key2] - storage: [key1, key2] - -# [opt] Labels for the project - merged with the ones defined in defaults -labels: - environment: dev2 - costcenter: apps - -# [opt] Org policy overrides defined at project level -org_policies: - compute.disableGuestAttributesAccess: - rules: - - enforce: true - compute.trustedImageProjects: - rules: - - allow: - values: - - projects/fast-dev-iac-core-0 - compute.vmExternalIpAccess: - rules: - - deny: - all: true - -# [opt] Prefix - overrides default if set -prefix: test1 - -# [opt] Service account to create for the project and their roles on the project -# in name => [roles] format -service_accounts: - another-service-account: - - roles/compute.admin - my-service-account: - - roles/compute.adminv1 - -# [opt] APIs to enable on the project. -services: - - storage.googleapis.com - - stackdriver.googleapis.com - - compute.googleapis.com - -# [opt] Roles to assign to the service identities in service => [roles] format -service_identities_iam: - compute: - - roles/storage.objectViewer - - # [opt] VPC setup. - # If set enables the `compute.googleapis.com` service and configures - # service project attachment -vpc: - # [opt] If set, enables the container API - gke_setup: - # Grants "roles/container.hostServiceAgentUser" to the container robot if set - enable_host_service_agent: false - - # Grants "roles/compute.securityAdmin" to the container robot if set - enable_security_admin: true - - # Host project the project will be service project of - host_project: fast-dev-net-spoke-0 - - # [opt] Services for which set up the IAM in the host project - service_iam_grants: - - dataproc.googleapis.com - - # [opt] Roles to rant service project service identities in host project - service_identity_iam: - "roles/compute.networkUser": - - cloudservices - - container-engine - - # [opt] Subnets in the host project where principals will be granted networkUser - # in region/subnet-name => [principals] - subnets_iam: - europe-west1/dev-default-ew1: - - user:foobar@example.com - - serviceAccount:my-service-account diff --git a/blueprints/factories/project-factory/variables.tf b/blueprints/factories/project-factory/variables.tf index f4dbc629..67917846 100644 --- a/blueprints/factories/project-factory/variables.tf +++ b/blueprints/factories/project-factory/variables.tf @@ -14,215 +14,84 @@ * limitations under the License. */ -variable "billing_account_id" { - description = "Billing account id." - type = string -} - -variable "billing_alert" { - description = "Billing alert configuration." +variable "data_defaults" { + description = "Optional default values used when corresponding project data from files are missing." type = object({ - amount = number - thresholds = object({ - current = list(number) - forecasted = list(number) - }) - credit_treatment = string + billing_account = optional(string) + contacts = optional(map(list(string)), {}) + labels = optional(map(string), {}) + metric_scopes = optional(list(string), []) + prefix = optional(string) + service_encryption_key_ids = optional(map(list(string)), {}) + service_perimeter_bridges = optional(list(string), []) + service_perimeter_standard = optional(string) + services = optional(list(string), []) + shared_vpc_service_config = optional(object({ + host_project = string + service_identity_iam = optional(map(list(string)), {}) + service_iam_grants = optional(list(string), []) + }), { host_project = null }) + tag_bindings = optional(map(string), {}) + # non-project resources + service_accounts = optional(map(object({ + default_roles = optional(bool, true) + })), {}) }) - default = null -} - -variable "defaults" { - description = "Project factory default values." - type = object({ - billing_account_id = string - billing_alert = object({ - amount = number - thresholds = object({ - current = list(number) - forecasted = list(number) - }) - credit_treatment = string - }) - environment_dns_zone = string - essential_contacts = list(string) - labels = map(string) - notification_channels = list(string) - shared_vpc_self_link = string - vpc_host_project = string - }) - default = null -} - -variable "descriptive_name" { - description = "Name of the project name. Used for project name instead of `name` variable." - type = string - default = null -} - -variable "dns_zones" { - description = "DNS private zones to create as child of var.defaults.environment_dns_zone." - type = list(string) - default = [] -} - -variable "essential_contacts" { - description = "Email contacts to be used for billing and GCP notifications." - type = list(string) - default = [] -} - -variable "folder_id" { - description = "Folder ID for the folder where the project will be created." - type = string - default = null -} - -variable "group_iam" { - description = "Custom IAM settings in group => [role] format." - type = map(list(string)) - default = {} -} - -variable "group_iam_additive" { - description = "Custom additive IAM settings in group => [role] format." - type = map(list(string)) - default = {} -} - -variable "iam" { - description = "Custom IAM settings in role => [principal] format." - type = map(list(string)) - default = {} -} - -variable "iam_additive" { - description = "Custom additive IAM settings in role => [principal] format." - type = map(list(string)) - default = {} -} - -variable "kms_service_agents" { - description = "KMS IAM configuration in as service => [key]." - type = map(list(string)) - default = {} -} - -variable "labels" { - description = "Labels to be assigned at project level." - type = map(string) - default = {} -} - -variable "org_policies" { - description = "Org-policy overrides at project level." - type = map(object({ - inherit_from_parent = optional(bool) # for list policies only. - reset = optional(bool) - rules = optional(list(object({ - allow = optional(object({ - all = optional(bool) - values = optional(list(string)) - })) - deny = optional(object({ - all = optional(bool) - values = optional(list(string)) - })) - enforce = optional(bool) # for boolean policies only. - condition = optional(object({ - description = optional(string) - expression = optional(string) - location = optional(string) - title = optional(string) - }), {}) - })), []) - })) + nullable = false default = {} - nullable = false } -variable "prefix" { - description = "Prefix used for resource names." - type = string - validation { - condition = var.prefix != "" - error_message = "Prefix cannot be empty." - } -} - -variable "project_id" { - description = "Project id." - type = string -} - -variable "service_accounts" { - description = "Service accounts to be created, and roles assigned them on the project." - type = map(list(string)) - default = {} -} - -variable "service_accounts_additive" { - description = "Service accounts to be created, and roles assigned them on the project additively." - type = map(list(string)) - default = {} -} - -variable "service_accounts_iam" { - description = "IAM bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}." - type = map(map(list(string))) - default = {} - nullable = false -} - -variable "service_accounts_iam_additive" { - description = "IAM additive bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}." - type = map(map(list(string))) - default = {} - nullable = false -} - -variable "service_identities_iam" { - description = "Custom IAM settings for service identities in service => [role] format." - type = map(list(string)) - default = {} - nullable = false -} - -variable "service_identities_iam_additive" { - description = "Custom additive IAM settings for service identities in service => [role] format." - type = map(list(string)) - default = {} - nullable = false -} - -variable "services" { - description = "Services to be enabled for the project." - type = list(string) - default = [] - nullable = false -} - -variable "vpc" { - description = "VPC configuration for the project." +variable "data_merges" { + description = "Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`." type = object({ - host_project = string - gke_setup = optional(object({ - enable_security_admin = optional(bool, false) - enable_host_service_agent = optional(bool, false) - }), {}) - service_iam_grants = optional(list(string), []) - service_identity_iam = optional(map(list(string)), {}) - subnets_iam = optional(map(list(string)), {}) + contacts = optional(map(list(string)), {}) + labels = optional(map(string), {}) + metric_scopes = optional(list(string), []) + service_encryption_key_ids = optional(map(list(string)), {}) + service_perimeter_bridges = optional(list(string), []) + services = optional(list(string), []) + tag_bindings = optional(map(string), {}) + # non-project resources + service_accounts = optional(map(object({ + default_roles = optional(bool, true) + })), {}) + }) + nullable = false + default = {} +} + +variable "data_overrides" { + description = "Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`." + type = object({ + billing_account = optional(string) + contacts = optional(map(list(string))) + prefix = optional(string) + service_encryption_key_ids = optional(map(list(string))) + service_perimeter_bridges = optional(list(string)) + service_perimeter_standard = optional(string) + tag_bindings = optional(map(string)) + services = optional(list(string)) + # non-project resources + service_accounts = optional(map(object({ + default_roles = optional(bool, true) + }))) + }) + nullable = false + default = {} +} + +variable "factory_data" { + description = "Project data from either YAML files or externally parsed data." + type = object({ + data = optional(map(any)) + data_path = optional(string) }) - default = { - host_project = null - } nullable = false validation { - condition = var.vpc.host_project != null || ( - var.vpc.host_project == null && length(var.vpc.gke_setup) == 0 && length(var.vpc.service_iam_grants) == 0 && - length(var.vpc.service_identity_iam) == 0 && length(var.vpc.subnets_iam) == 0 - ) - error_message = "host_project is required if providing any additional configuration for vpc" + condition = ( + (var.factory_data.data != null ? 1 : 0) + + (var.factory_data.data_path != null ? 1 : 0) + ) == 1 + error_message = "One of data or data_path needs to be set." } } diff --git a/blueprints/networking/hub-and-spoke-peering/main.tf b/blueprints/networking/hub-and-spoke-peering/main.tf index 95007ab7..98aac6fb 100644 --- a/blueprints/networking/hub-and-spoke-peering/main.tf +++ b/blueprints/networking/hub-and-spoke-peering/main.tf @@ -32,9 +32,11 @@ module "project" { source = "../../../modules/project" project_create = var.project_create != null billing_account = try(var.project_create.billing_account, null) - oslogin = try(var.project_create.oslogin, false) - parent = try(var.project_create.parent, null) - name = var.project_id + compute_metadata = var.project_create.oslogin != true ? {} : { + enable-oslogin = "true" + } + parent = try(var.project_create.parent, null) + name = var.project_id services = [ "compute.googleapis.com", "container.googleapis.com" diff --git a/blueprints/networking/shared-vpc-gke/main.tf b/blueprints/networking/shared-vpc-gke/main.tf index dafb2704..302ce735 100644 --- a/blueprints/networking/shared-vpc-gke/main.tf +++ b/blueprints/networking/shared-vpc-gke/main.tf @@ -41,8 +41,9 @@ module "project-svc-gce" { prefix = var.prefix name = "gce" services = var.project_services - oslogin = true - oslogin_admins = var.owners_gce + compute_metadata = { + enable-oslogin = "true" + } shared_vpc_service_config = { host_project = module.project-host.project_id service_identity_iam = { @@ -50,7 +51,8 @@ module "project-svc-gce" { } } iam = { - "roles/owner" = var.owners_gce + "roles/compute.osAdminLogin" = var.owners_gce + "roles/owner" = var.owners_gce } } diff --git a/blueprints/third-party-solutions/wordpress/cloudrun/README.md b/blueprints/third-party-solutions/wordpress/cloudrun/README.md index 0ffcc395..60754e4b 100644 --- a/blueprints/third-party-solutions/wordpress/cloudrun/README.md +++ b/blueprints/third-party-solutions/wordpress/cloudrun/README.md @@ -10,7 +10,13 @@ This architecture can be used for the following use cases and more: * Intranet / internal Wiki * E-commerce platform -# Architecture +## TODO + +* [ ] refactor variables merging WP configuration in a single variable +* [ ] optional creation of a remote artifact registry repository +* [ ] optional serverless connector + +## Architecture ![Wordpress on Cloud Run](images/architecture.png "Wordpress on Cloud Run") @@ -20,9 +26,7 @@ The main components that are deployed in this architecture are the following (yo * [Cloud SQL](https://cloud.google.com/sql): Managed solution for SQL databases * [VPC Serverless Connector](https://cloud.google.com/vpc/docs/serverless-vpc-access): Solution to access the CloudSQL VPC from Cloud Run, using only internal IP addresses -# Setup - -## Prerequisites +## Setup and deployment ### Setting up the project for the deployment @@ -30,8 +34,6 @@ This example will deploy all its resources into the project defined by the `proj If `project_create` is left to null, the identity performing the deployment needs the `owner` role on the project defined by the `project_id` variable. Otherwise, the identity performing the deployment needs `resourcemanager.projectCreator` on the resource hierarchy node specified by `project_create.parent` and `billing.user` on the billing account specified by `project_create.billing_account_id`. -## Deployment - ### Step 0: Cloning the repository If you want to deploy from your Cloud Shell, click on the image below, sign in if required and when the prompt appears, click on “confirm”. @@ -116,20 +118,19 @@ terraform destroy The above command will delete the associated resources so there will be no billable charges made afterwards. - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L57) | Prefix used for resource names. | string | ✓ | | +| [prefix](variables.tf#L63) | Prefix used for resource names. | string | ✓ | | | [project_id](variables.tf#L81) | Project id, references existing project if `project_create` is null. | string | ✓ | | | [wordpress_image](variables.tf#L92) | Image to run with Cloud Run, starts with \"gcr.io\". | string | ✓ | | -| [cloud_run_invoker](variables.tf#L18) | IAM member authorized to access the end-point (for example, 'user:YOUR_IAM_USER' for only you or 'allUsers' for everyone). | string | | "allUsers" | -| [cloudsql_password](variables.tf#L24) | CloudSQL password (will be randomly generated by default). | string | | null | -| [connector](variables.tf#L30) | Existing VPC serverless connector to use if not creating a new one. | string | | null | -| [create_connector](variables.tf#L36) | Should a VPC serverless connector be created or not. | bool | | true | -| [ip_ranges](variables.tf#L43) | CIDR blocks: VPC serverless connector, Private Service Access(PSA) for CloudSQL, CloudSQL VPC. | object({…}) | | {…} | -| [principals](variables.tf#L66) | List of users to give rights to (CloudSQL admin, client and instanceUser, Logging admin, Service Account User and TokenCreator), eg 'user@domain.com'. | list(string) | | [] | +| [admin_principal](variables.tf#L17) | User or group that is assigned roles, in IAM format (`group:foo@example.com`). | string | | null | +| [cloud_run_invoker](variables.tf#L24) | IAM member authorized to access the end-point (for example, 'user:YOUR_IAM_USER' for only you or 'allUsers' for everyone). | string | | "allUsers" | +| [cloudsql_password](variables.tf#L30) | CloudSQL password (will be randomly generated by default). | string | | null | +| [connector](variables.tf#L36) | Existing VPC serverless connector to use if not creating a new one. | string | | null | +| [create_connector](variables.tf#L42) | Should a VPC serverless connector be created or not. | bool | | true | +| [ip_ranges](variables.tf#L49) | CIDR blocks: VPC serverless connector, Private Service Access(PSA) for CloudSQL, CloudSQL VPC. | object({…}) | | {…} | | [project_create](variables.tf#L72) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | | [region](variables.tf#L86) | Region for the created resources. | string | | "europe-west4" | | [wordpress_password](variables.tf#L97) | Password for the Wordpress user (will be randomly generated by default). | string | | null | @@ -143,5 +144,20 @@ The above command will delete the associated resources so there will be no billa | [cloudsql_password](outputs.tf#L23) | CloudSQL password. | ✓ | | [wp_password](outputs.tf#L29) | Wordpress user password. | ✓ | | [wp_user](outputs.tf#L35) | Wordpress username. | | - +## Test + +```hcl +module "test" { + source = "./fabric/blueprints/third-party-solutions/wordpress/cloudrun" + admin_principal = "group:foo@example.com" + prefix = "wp-cr-test" + project_create = { + billing_account_id = "1234-ABCD-1234" + parent = "folders/1234563" + } + project_id = "test-prj" + wordpress_image = "gcr.io/myprj/wordpress" +} +# tftest modules=5 resources=33 +``` diff --git a/blueprints/third-party-solutions/wordpress/cloudrun/main.tf b/blueprints/third-party-solutions/wordpress/cloudrun/main.tf index 04027790..2e089b79 100644 --- a/blueprints/third-party-solutions/wordpress/cloudrun/main.tf +++ b/blueprints/third-party-solutions/wordpress/cloudrun/main.tf @@ -16,30 +16,29 @@ locals { - all_principals_iam = [for k in var.principals : "user:${k}"] cloudsql_conf = { database_version = "MYSQL_8_0" tier = "db-g1-small" db = "wp-mysql" user = "admin" } - iam = { - # CloudSQL - "roles/cloudsql.admin" = local.all_principals_iam - "roles/cloudsql.client" = local.all_principals_iam - "roles/cloudsql.instanceUser" = local.all_principals_iam - # common roles - "roles/logging.admin" = local.all_principals_iam - "roles/iam.serviceAccountUser" = local.all_principals_iam - "roles/iam.serviceAccountTokenCreator" = local.all_principals_iam - } + iam_roles = [ + "roles/cloudsql.admin", + "roles/cloudsql.client", + "roles/cloudsql.instanceUser", + "roles/logging.admin", + "roles/iam.serviceAccountUser", + "roles/iam.serviceAccountTokenCreator" + ] connector = var.connector == null ? google_vpc_access_connector.connector.0.self_link : var.connector wp_user = "user" wp_pass = var.wordpress_password == null ? random_password.wp_password.result : var.wordpress_password } +resource "random_password" "wp_password" { + length = 8 +} -# either create a project or set up the given one module "project" { source = "../../../../modules/project" name = var.project_id @@ -47,8 +46,19 @@ module "project" { billing_account = try(var.project_create.billing_account_id, null) project_create = var.project_create != null prefix = var.project_create == null ? null : var.prefix - iam = var.project_create != null ? local.iam : {} - iam_additive = var.project_create == null ? local.iam : {} + iam = ( + var.project_create != true || var.admin_principal == null + ? {} + : { for r in local.iam_roles : r => [var.admin_principal] } + ) + iam_bindings_additive = ( + var.project_create == true || var.admin_principal == null + ? {} + : { for r in local.iam_roles : r => { + member = var.admin_principal + role = r + } } + ) services = [ "run.googleapis.com", "logging.googleapis.com", @@ -60,60 +70,45 @@ module "project" { ] } - -resource "random_password" "wp_password" { - length = 8 -} - - -# create the Cloud Run service module "cloud_run" { source = "../../../../modules/cloud-run" project_id = module.project.project_id name = "${var.prefix}-cr-wordpress" region = var.region - - containers = [{ - image = var.wordpress_image - ports = [{ - name = "http1" - protocol = null - container_port = var.wordpress_port - }] - options = { - command = null - args = null - env_from = null - # set up the database connection + containers = { + wp = { + image = var.wordpress_image + ports = { + http = { container_port = var.wordpress_port } + } env = { - "APACHE_HTTP_PORT_NUMBER" : var.wordpress_port - "WORDPRESS_DATABASE_HOST" : module.cloudsql.ip - "WORDPRESS_DATABASE_NAME" : local.cloudsql_conf.db - "WORDPRESS_DATABASE_USER" : local.cloudsql_conf.user - "WORDPRESS_DATABASE_PASSWORD" : var.cloudsql_password == null ? module.cloudsql.user_passwords[local.cloudsql_conf.user] : var.cloudsql_password - "WORDPRESS_USERNAME" : local.wp_user - "WORDPRESS_PASSWORD" : local.wp_pass + APACHE_HTTP_PORT_NUMBER = var.wordpress_port + WORDPRESS_DATABASE_HOST = module.cloudsql.ip + WORDPRESS_DATABASE_NAME = local.cloudsql_conf.db + WORDPRESS_DATABASE_USER = local.cloudsql_conf.user + WORDPRESS_DATABASE_PASSWORD = ( + var.cloudsql_password == null + ? module.cloudsql.user_passwords[local.cloudsql_conf.user] + : var.cloudsql_password + ) + WORDPRESS_USERNAME = local.wp_user + WORDPRESS_PASSWORD = local.wp_pass } } - resources = null - volume_mounts = null - }] - + } iam = { "roles/run.invoker" : [var.cloud_run_invoker] } - revision_annotations = { autoscaling = { min_scale = 1 max_scale = 2 } # connect to CloudSQL - cloudsql_instances = [module.cloudsql.connection_name] - vpcaccess_connector = null + cloudsql_instances = [module.cloudsql.connection_name] # allow all traffic - vpcaccess_egress = "all-traffic" vpcaccess_connector = local.connector + vpcaccess_egress = "all-traffic" } ingress_settings = "all" } diff --git a/blueprints/third-party-solutions/wordpress/cloudrun/variables.tf b/blueprints/third-party-solutions/wordpress/cloudrun/variables.tf index abb00d2d..4956f2f4 100644 --- a/blueprints/third-party-solutions/wordpress/cloudrun/variables.tf +++ b/blueprints/third-party-solutions/wordpress/cloudrun/variables.tf @@ -14,6 +14,12 @@ * limitations under the License. */ +variable "admin_principal" { + description = "User or group that is assigned roles, in IAM format (`group:foo@example.com`)." + type = string + default = null +} + # Documentation: https://cloud.google.com/run/docs/securing/managing-access#making_a_service_public variable "cloud_run_invoker" { type = string @@ -63,12 +69,6 @@ variable "prefix" { } } -variable "principals" { - description = "List of users to give rights to (CloudSQL admin, client and instanceUser, Logging admin, Service Account User and TokenCreator), eg 'user@domain.com'." - type = list(string) - default = [] -} - variable "project_create" { description = "Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." type = object({ diff --git a/fast/stages-multitenant/0-bootstrap-tenant/README.md b/fast/stages-multitenant/0-bootstrap-tenant/README.md index f831f9af..96747b21 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/README.md +++ b/fast/stages-multitenant/0-bootstrap-tenant/README.md @@ -186,7 +186,7 @@ This configuration is possible but unsupported and only exists for development p | [identity-providers.tf](./identity-providers.tf) | Workload Identity Federation provider definitions. | | google_iam_workload_identity_pool · google_iam_workload_identity_pool_provider | | [log-export.tf](./log-export.tf) | Audit log project and sink. | bigquery-dataset · gcs · logging-bucket · project · pubsub | | | [main.tf](./main.tf) | Module-level locals and resources. | folder | | -| [organization.tf](./organization.tf) | Organization tag and conditional IAM grant. | organization | google_organization_iam_member · google_tags_tag_value_iam_member | +| [organization.tf](./organization.tf) | Organization tag and conditional IAM grant. | organization | google_tags_tag_value_iam_member | | [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file | | [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | | google_storage_bucket_object | | [outputs.tf](./outputs.tf) | Module outputs. | | | @@ -198,24 +198,25 @@ This configuration is possible but unsupported and only exists for development p |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L20) | Automation resources created by the organization-level bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | | [billing_account](variables.tf#L39) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | object({…}) | ✓ | | | -| [organization](variables.tf#L192) | Organization details. | object({…}) | ✓ | | 0-bootstrap | -| [prefix](variables.tf#L208) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | -| [tag_keys](variables.tf#L231) | Organization tag keys. | object({…}) | ✓ | | 1-resman | -| [tag_names](variables.tf#L242) | Customized names for resource management tags. | object({…}) | ✓ | | 1-resman | -| [tag_values](variables.tf#L253) | Organization resource management tag values. | map(string) | ✓ | | 1-resman | -| [tenant_config](variables.tf#L260) | Tenant configuration. Short name must be 4 characters or less. If `short_name_is_prefix` is true, short name must be 9 characters or less, and will be used as the prefix as is. | object({…}) | ✓ | | | +| [organization](variables.tf#L214) | Organization details. | object({…}) | ✓ | | 0-bootstrap | +| [prefix](variables.tf#L230) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | +| [tag_keys](variables.tf#L253) | Organization tag keys. | object({…}) | ✓ | | 1-resman | +| [tag_names](variables.tf#L264) | Customized names for resource management tags. | object({…}) | ✓ | | 1-resman | +| [tag_values](variables.tf#L275) | Organization resource management tag values. | map(string) | ✓ | | 1-resman | +| [tenant_config](variables.tf#L282) | Tenant configuration. Short name must be 4 characters or less. If `short_name_is_prefix` is true, short name must be 9 characters or less, and will be used as the prefix as is. | object({…}) | ✓ | | | | [cicd_repositories](variables.tf#L49) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | | [custom_roles](variables.tf#L95) | Custom roles defined at the organization level, in key => id format. | object({…}) | | null | 0-bootstrap | | [fast_features](variables.tf#L105) | Selective control for top-level FAST features. | object({…}) | | {} | 0-bootstrap | | [federated_identity_providers](variables.tf#L119) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | | [group_iam](variables.tf#L133) | Tenant-level custom group IAM settings in group => [roles] format. | map(list(string)) | | {} | | -| [iam](variables.tf#L139) | Tenant-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | -| [iam_additive](variables.tf#L145) | Tenant-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | -| [locations](variables.tf#L151) | Optional locations for GCS, BigQuery, and logging buckets created here. These are the defaults set at the organization level, and can be overridden via the tenant config variable. | object({…}) | | {…} | 0-bootstrap | -| [log_sinks](variables.tf#L171) | Tenant-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | -| [outputs_location](variables.tf#L202) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | -| [project_parent_ids](variables.tf#L218) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the tenant folder as parent. | object({…}) | | {…} | | -| [test_principal](variables.tf#L301) | Used when testing to bypass the data source returning the current identity. | string | | null | | +| [groups](variables.tf#L139) | Group names or emails to grant organization-level permissions. If just the name is provided, the default organization domain is assumed. | object({…}) | | {} | 0-bootstrap | +| [iam](variables.tf#L152) | Tenant-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | +| [iam_bindings_additive](variables.tf#L158) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | | +| [locations](variables.tf#L173) | Optional locations for GCS, BigQuery, and logging buckets created here. These are the defaults set at the organization level, and can be overridden via the tenant config variable. | object({…}) | | {…} | 0-bootstrap | +| [log_sinks](variables.tf#L193) | Tenant-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | +| [outputs_location](variables.tf#L224) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | +| [project_parent_ids](variables.tf#L240) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the tenant folder as parent. | object({…}) | | {…} | | +| [test_principal](variables.tf#L323) | Used when testing to bypass the data source returning the current identity. | string | | null | | ## Outputs diff --git a/fast/stages-multitenant/0-bootstrap-tenant/automation-sas.tf b/fast/stages-multitenant/0-bootstrap-tenant/automation-sas.tf index 453b6b8d..fd86fc47 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/automation-sas.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/automation-sas.tf @@ -106,6 +106,7 @@ module "automation-tf-resman-sa-stage2-3" { } # assign org policy admin with a tag-based condition to stage 2 and 3 SAs +# TODO: move to new iam_bindings_additive in the organization module resource "google_organization_iam_member" "org_policy_admin_stage2_3" { for_each = { diff --git a/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf b/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf index fd0ec167..c8d13f53 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/cicd.tf @@ -60,6 +60,9 @@ locals { ) ) } + cicd_sa_resman = try( + module.automation-tf-cicd-sa-bootstrap["0"].iam_email, null + ) } # tenant bootstrap runs in the org scope and uses top-level automation project @@ -145,10 +148,11 @@ module "automation-tf-org-resman-sa" { project_id = var.automation.project_id name = local.resman_sa service_account_create = false - iam_additive = { - "roles/iam.serviceAccountTokenCreator" = compact([ - try(module.automation-tf-cicd-sa-bootstrap["0"].iam_email, null) - ]) + iam_bindings_additive = local.cicd_sa_resman == null ? {} : { + sa_resman_cicd = { + member = local.cicd_sa_resman + role = "roles/iam.serviceAccountTokenCreator" + } } } diff --git a/fast/stages-multitenant/0-bootstrap-tenant/main.tf b/fast/stages-multitenant/0-bootstrap-tenant/main.tf index 5f1c630b..011999e3 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/main.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/main.tf @@ -95,6 +95,6 @@ module "tenant-folder-iam" { module.automation-tf-resman-sa.iam_email ] }) - iam_additive = var.iam_additive - depends_on = [module.automation-project] + iam_bindings_additive = var.iam_bindings_additive + depends_on = [module.automation-project] } diff --git a/fast/stages-multitenant/0-bootstrap-tenant/organization.tf b/fast/stages-multitenant/0-bootstrap-tenant/organization.tf index f08f0fba..342ccc37 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/organization.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/organization.tf @@ -26,19 +26,45 @@ locals { module "organization" { source = "../../../modules/organization" organization_id = "organizations/${var.organization.id}" - iam_additive = merge( + iam_bindings_additive = merge( { - "roles/resourcemanager.organizationViewer" = [ - "group:${local.groups.gcp-admins}" - ] + admins_org_viewer = { + member = "group:${local.groups.gcp-admins}" + role = "roles/resourcemanager.organizationViewer" + } + admins_org_policy_admin = { + member = "group:${local.groups.gcp-admins}" + role = "roles/orgpolicy.policyAdmin" + condition = { + title = "org_policy_tag_${var.tenant_config.short_name}_scoped_admins" + description = "Org policy tag scoped grant for tenant ${var.tenant_config.short_name}." + expression = local.iam_tenant_condition + } + } + sa_resman_org_policy_admin = { + member = module.automation-tf-resman-sa.iam_email + role = "roles/orgpolicy.policyAdmin" + condition = { + title = "org_policy_tag_${var.tenant_config.short_name}_scoped_sa_resman" + description = "Org policy tag scoped grant for tenant ${var.tenant_config.short_name}." + expression = local.iam_tenant_condition + } + } }, - local.billing_mode == "org" ? { - "roles/billing.admin" = [ - "group:${local.groups.gcp-admins}", - module.automation-tf-resman-sa.iam_email - ] - "roles/billing.costsManager" = ["group:${local.groups.gcp-admins}"] - } : {} + local.billing_mode != "org" ? {} : { + admins_billing_admin = { + member = "group:${local.groups.gcp-admins}" + role = "roles/billing.admin" + } + admins_billing_costs_manager = { + member = "group:${local.groups.gcp-admins}" + role = "roles/billing.costsManager" + } + sa_resman_billing_admin = { + member = module.automation-tf-resman-sa.iam_email + role = "roles/billing.admin" + } + } ) tags = { tenant = { @@ -50,6 +76,8 @@ module "organization" { } } +# TODO: use tag IAM with id in the organization module + resource "google_tags_tag_value_iam_member" "resman_tag_user" { for_each = var.tag_values tag_value = each.value @@ -64,21 +92,4 @@ resource "google_tags_tag_value_iam_member" "admins_tag_viewer" { member = "group:${local.groups.gcp-admins}" } -# assign org policy admin with a tag-based condition to admin group and stage 1 SA - -resource "google_organization_iam_member" "org_policy_admin_stage0" { - for_each = toset([ - "group:${local.groups.gcp-admins}", - module.automation-tf-resman-sa.iam_email - ]) - org_id = var.organization.id - role = "roles/orgpolicy.policyAdmin" - member = each.key - condition { - title = "org_policy_tag_${var.tenant_config.short_name}_scoped" - description = "Org policy tag scoped grant for tenant ${var.tenant_config.short_name}." - expression = local.iam_tenant_condition - } -} - # tag-based condition for service accounts is in the automation-sa file diff --git a/fast/stages-multitenant/0-bootstrap-tenant/variables.tf b/fast/stages-multitenant/0-bootstrap-tenant/variables.tf index af56db39..18594c40 100644 --- a/fast/stages-multitenant/0-bootstrap-tenant/variables.tf +++ b/fast/stages-multitenant/0-bootstrap-tenant/variables.tf @@ -136,16 +136,38 @@ variable "group_iam" { default = {} } +variable "groups" { + # tfdoc:variable:source 0-bootstrap + # https://cloud.google.com/docs/enterprise/setup-checklist + description = "Group names or emails to grant organization-level permissions. If just the name is provided, the default organization domain is assumed." + type = object({ + gcp-devops = optional(string) + gcp-network-admins = optional(string) + gcp-security-admins = optional(string) + }) + default = {} + nullable = false +} + variable "iam" { description = "Tenant-level custom IAM settings in role => [principal] format." type = map(list(string)) default = {} } -variable "iam_additive" { - description = "Tenant-level custom IAM settings in role => [principal] format for non-authoritative bindings." - type = map(list(string)) - default = {} +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } variable "locations" { diff --git a/fast/stages-multitenant/1-resman-tenant/root_node.tf b/fast/stages-multitenant/1-resman-tenant/root_node.tf index 5b83d2dd..7b02a609 100644 --- a/fast/stages-multitenant/1-resman-tenant/root_node.tf +++ b/fast/stages-multitenant/1-resman-tenant/root_node.tf @@ -26,16 +26,19 @@ module "root-folder" { ) name = var.test_skip_data_sources ? "Test" : null # end test attributes - iam_additive = { - "roles/accesscontextmanager.policyAdmin" = [ - local.automation_sas_iam.security - ] - "roles/compute.orgFirewallPolicyAdmin" = [ - local.automation_sas_iam.networking - ] - "roles/compute.xpnAdmin" = [ - local.automation_sas_iam.networking - ] + iam_bindings_additive = { + sa_net_fw_policy_admin = { + member = local.automation_sas_iam.networking + role = "roles/compute.orgFirewallPolicyAdmin" + } + sa_net_xpn_admin = { + member = local.automation_sas_iam.networking + role = "roles/compute.xpnAdmin" + } + sa_sec_vpcsc_admin = { + member = local.automation_sas_iam.security + role = "roles/accesscontextmanager.policyAdmin" + } } org_policies_data_path = var.organization_policy_data_path } diff --git a/fast/stages/0-bootstrap/README.md b/fast/stages/0-bootstrap/README.md index ffaeb17d..9bfc7506 100644 --- a/fast/stages/0-bootstrap/README.md +++ b/fast/stages/0-bootstrap/README.md @@ -357,11 +357,20 @@ In code, the distinction above reflects on how IAM bindings are specified in the This makes it easy to tweak user roles by adding mappings to the `iam_groups` variables of the relevant resources, without having to understand and deal with the details of service account roles. -In those cases where roles need to be assigned to end-user service accounts (e.g. an application or pipeline service account), we offer a stage-level `iam` variable that allows pinpointing individual role/members pairs, without having to touch the code internals, to avoid the risk of breaking a critical role for a robot account. The variable can also be used to assign roles to specific users or to groups external to the organization, e.g. to support external suppliers. +One more critical difference in IAM bindings is between authoritative and additive: -The one exception to this convention is for roles which are part of the delegated grant condition described above, and which can then be assigned from other stages. In this case, use the `iam_additive` variable as they are implemented with non-authoritative resources. Using non-authoritative bindings ensure that re-executing this stage will not override any bindings set in downstream stages. +- authoritative bindings have complete control on principals for a given role; this is the recommended best practice when a single automation actor controls the role, as it removes drift each time Terraform runs +- additive bindings have control only on given role/principal pairs, and need to be used whenever multiple automation actors need to control the role, as is the case for the network user role in Shared VPC setups, and many other situations -A full reference of IAM roles managed by this stage [is available here](./IAM.md). +This stage groups all IAM definitions in the [organization-iam.tf](./organization-iam.tf) file, to allow easy parsing of roles assigned to each group and machine identity. + +When customizations are needed, three stage-level variables allow injecting additional bindings to match the desired setup: + +- `group_iam` allows adding authoritative bindings for groups +- `iam` allows adding authoritative bindings for any type of supported principal, and is merged with the internal `iam` local and then with group bindings at the module level +- `iam_bindings_additive` allows adding individual role/member pairs, and also supports IAM conditions + +Refer to the [project module](../../../modules/project/) for examples on how to use the IAM variables, and they are an interface shared across all our modules. ### Log sinks and log destinations @@ -498,7 +507,8 @@ The remaining configuration is manual, as it regards the repositories themselves | [identity-providers.tf](./identity-providers.tf) | Workload Identity Federation provider definitions. | | google_iam_workload_identity_pool · google_iam_workload_identity_pool_provider | | [log-export.tf](./log-export.tf) | Audit log project and sink. | bigquery-dataset · gcs · logging-bucket · project · pubsub | | | [main.tf](./main.tf) | Module-level locals and resources. | | | -| [organization.tf](./organization.tf) | Organization-level IAM. | organization | google_organization_iam_binding | +| [organization-iam.tf](./organization-iam.tf) | Organization-level IAM bindings locals. | | | +| [organization.tf](./organization.tf) | Organization-level IAM. | organization | | | [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file | | [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | | google_storage_bucket_object | | [outputs.tf](./outputs.tf) | Module outputs. | | | @@ -509,21 +519,22 @@ The remaining configuration is manual, as it regards the repositories themselves | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [billing_account](variables.tf#L17) | Billing account id. If billing account is not part of the same org set `is_org_level` to `false`. To disable handling of billing IAM roles set `no_iam` to `true`. | object({…}) | ✓ | | | -| [organization](variables.tf#L206) | Organization details. | object({…}) | ✓ | | | -| [prefix](variables.tf#L221) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | | +| [organization](variables.tf#L219) | Organization details. | object({…}) | ✓ | | | +| [prefix](variables.tf#L234) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | | | [bootstrap_user](variables.tf#L27) | Email of the nominal user running this stage for the first time. | string | | null | | | [cicd_repositories](variables.tf#L33) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | | [custom_role_names](variables.tf#L79) | Names of custom roles defined at the org level. | object({…}) | | {…} | | | [custom_roles](variables.tf#L93) | Map of role names => list of permissions to additionally create at the organization level. | map(list(string)) | | {} | | | [fast_features](variables.tf#L100) | Selective control for top-level FAST features. | object({…}) | | {} | | | [federated_identity_providers](variables.tf#L113) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | -| [groups](variables.tf#L132) | Group names or emails to grant organization-level permissions. If just the name is provided, the default organization domain is assumed. | map(string) | | {…} | | -| [iam](variables.tf#L150) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | -| [iam_additive](variables.tf#L156) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | -| [locations](variables.tf#L162) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | | -| [log_sinks](variables.tf#L181) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | -| [outputs_location](variables.tf#L215) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | -| [project_parent_ids](variables.tf#L230) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the organization as parent. | object({…}) | | {…} | | +| [group_iam](variables.tf#L132) | Organization-level authoritative IAM binding for groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | +| [groups](variables.tf#L140) | Group names or emails to grant organization-level permissions. If just the name is provided, the default organization domain is assumed. | map(string) | | {…} | | +| [iam](variables.tf#L158) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | +| [iam_bindings_additive](variables.tf#L165) | Organization-level custom additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | | +| [locations](variables.tf#L180) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {} | | +| [log_sinks](variables.tf#L194) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | +| [outputs_location](variables.tf#L228) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | +| [project_parent_ids](variables.tf#L243) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the organization as parent. | object({…}) | | {…} | | ## Outputs diff --git a/fast/stages/0-bootstrap/automation.tf b/fast/stages/0-bootstrap/automation.tf index 90c14a81..d74e9efb 100644 --- a/fast/stages/0-bootstrap/automation.tf +++ b/fast/stages/0-bootstrap/automation.tf @@ -16,6 +16,10 @@ # tfdoc:file:description Automation project and resources. +locals { + cicd_resman_sa = try(module.automation-tf-cicd-sa["resman"].iam_email, "") +} + module "automation-project" { source = "../../../modules/project" billing_account = var.billing_account.id @@ -151,11 +155,14 @@ module "automation-tf-resman-sa" { prefix = local.prefix # allow SA used by CI/CD workflow to impersonate this SA # we use additive IAM to allow tenant CI/CD SAs to impersonate it - iam_additive = { - "roles/iam.serviceAccountTokenCreator" = compact([ - try(module.automation-tf-cicd-sa["resman"].iam_email, null) - ]) - } + iam_bindings_additive = ( + local.cicd_resman_sa == "" ? {} : { + cicd_token_creator = { + member = local.cicd_resman_sa + role = "roles/iam.serviceAccountTokenCreator" + } + } + ) iam_storage_roles = { (module.automation-tf-output-gcs.name) = ["roles/storage.admin"] } diff --git a/fast/stages/0-bootstrap/organization-iam.tf b/fast/stages/0-bootstrap/organization-iam.tf new file mode 100644 index 00000000..c93bd540 --- /dev/null +++ b/fast/stages/0-bootstrap/organization-iam.tf @@ -0,0 +1,153 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Organization-level IAM bindings locals. + +locals { + # IAM roles in the org to reset (remove principals) + iam_delete_roles = [ + "roles/billing.creator" + ] + # domain IAM bindings + iam_domain_bindings = { + "domain:${var.organization.domain}" = { + authoritative = ["roles/browser"] + additive = [] + } + } + # human (groups) IAM bindings + iam_group_bindings = { + (local.groups.gcp-billing-admins) = { + authoritative = [] + additive = ( + local.billing_mode != "org" ? [] : [ + "roles/billing.admin", + "roles/billing.costsManager" + ] + ) + } + (local.groups.gcp-network-admins) = { + authoritative = [ + "roles/cloudasset.owner", + "roles/cloudsupport.techSupportEditor", + ] + additive = [ + "roles/compute.orgFirewallPolicyAdmin", + "roles/compute.xpnAdmin" + ] + } + (local.groups.gcp-organization-admins) = { + authoritative = [ + "roles/cloudasset.owner", + "roles/cloudsupport.admin", + "roles/compute.osAdminLogin", + "roles/compute.osLoginExternalUser", + "roles/owner", + "roles/resourcemanager.folderAdmin", + "roles/resourcemanager.organizationAdmin", + "roles/resourcemanager.projectCreator", + ] + additive = concat( + [ + "roles/orgpolicy.policyAdmin" + ], + local.billing_mode != "org" ? [] : [ + "roles/billing.admin", + "roles/billing.costsManager" + ] + ) + } + (local.groups.gcp-security-admins) = { + authoritative = [ + "roles/cloudasset.owner", + "roles/cloudsupport.techSupportEditor", + "roles/iam.securityReviewer", + "roles/logging.admin", + "roles/securitycenter.admin", + ] + additive = [ + "roles/accesscontextmanager.policyAdmin", + "roles/iam.organizationRoleAdmin", + "roles/orgpolicy.policyAdmin" + ] + } + (local.groups.gcp-support) = { + authoritative = [ + "roles/cloudsupport.techSupportEditor", + "roles/logging.viewer", + "roles/monitoring.viewer", + ] + additive = [] + } + } + # machine (service accounts) IAM bindings, in logical format + # the service account module's "magic" outputs allow us to use dynamic values + iam_sa_bindings = { + (module.automation-tf-bootstrap-sa.iam_email) = { + authoritative = [ + "roles/logging.admin", + "roles/resourcemanager.organizationAdmin", + "roles/resourcemanager.projectCreator", + "roles/resourcemanager.projectMover", + ] + additive = concat( + [ + "roles/iam.organizationRoleAdmin", + "roles/orgpolicy.policyAdmin" + ], + local.billing_mode != "org" ? [] : [ + "roles/billing.admin", + "roles/billing.costsManager" + ] + ) + } + (module.automation-tf-resman-sa.iam_email) = { + authoritative = [ + "roles/logging.admin", + "roles/resourcemanager.folderAdmin", + "roles/resourcemanager.projectCreator", + "roles/resourcemanager.tagAdmin", + "roles/resourcemanager.tagUser" + ] + additive = concat( + [ + "roles/orgpolicy.policyAdmin" + ], + local.billing_mode != "org" ? [] : [ + "roles/billing.admin", + "roles/billing.costsManager" + ] + ) + } + } + # bootstrap user bindings + iam_user_bootstrap_bindings = var.bootstrap_user == null ? {} : { + "user:${var.bootstrap_user}" = { + authoritative = [ + "roles/logging.admin", + "roles/owner", + "roles/resourcemanager.organizationAdmin", + "roles/resourcemanager.projectCreator" + ] + additive = ( + local.billing_mode != "org" ? [] : [ + "roles/billing.admin", + "roles/billing.costsManager" + ] + ) + } + } +} diff --git a/fast/stages/0-bootstrap/organization.tf b/fast/stages/0-bootstrap/organization.tf index 0e5fe0a8..0c20a4a3 100644 --- a/fast/stages/0-bootstrap/organization.tf +++ b/fast/stages/0-bootstrap/organization.tf @@ -17,111 +17,51 @@ # tfdoc:file:description Organization-level IAM. locals { - # organization authoritative IAM bindings, in an easy to edit format before - # they are combined with var.iam a bit further in locals - _iam = { - "roles/billing.creator" = [] - "roles/browser" = [ - "domain:${var.organization.domain}" - ] - "roles/logging.admin" = concat( - [ - module.automation-tf-bootstrap-sa.iam_email, - module.automation-tf-resman-sa.iam_email - ], - local._iam_bootstrap_user - ) - "roles/owner" = local._iam_bootstrap_user - "roles/resourcemanager.folderAdmin" = [ - module.automation-tf-resman-sa.iam_email - ] - "roles/resourcemanager.organizationAdmin" = concat( - [module.automation-tf-bootstrap-sa.iam_email], - local._iam_bootstrap_user - ) - "roles/resourcemanager.projectCreator" = concat( - [ - module.automation-tf-bootstrap-sa.iam_email, - module.automation-tf-resman-sa.iam_email - ], - local._iam_bootstrap_user - ) - "roles/resourcemanager.projectMover" = [ - module.automation-tf-bootstrap-sa.iam_email - ] - "roles/resourcemanager.tagAdmin" = [ - module.automation-tf-resman-sa.iam_email - ] - "roles/resourcemanager.tagUser" = [ - module.automation-tf-resman-sa.iam_email - ] - } - # organization additive IAM bindings, in an easy to edit format before - # they are combined with var.iam_additive a bit further in locals - _iam_additive = merge( + # reassemble logical bindings into the formats expected by the module + _iam_bindings = merge( + local.iam_domain_bindings, + local.iam_sa_bindings, + local.iam_user_bootstrap_bindings, { - "roles/accesscontextmanager.policyAdmin" = [ - local.groups_iam.gcp-security-admins - ] - "roles/compute.orgFirewallPolicyAdmin" = [ - local.groups_iam.gcp-network-admins - ] - "roles/compute.xpnAdmin" = [ - local.groups_iam.gcp-network-admins - ] - # use additive to support cross-org roles for billing - "roles/iam.organizationRoleAdmin" = [ - # uncomment if roles/owner is removed to organization admins - # local.groups.gcp-organization-admins, - local.groups_iam.gcp-security-admins, - module.automation-tf-bootstrap-sa.iam_email - ] - "roles/orgpolicy.policyAdmin" = [ - local.groups_iam.gcp-organization-admins, - local.groups_iam.gcp-security-admins, - module.automation-tf-resman-sa.iam_email - ] - # the following is useful if roles/browser is not desirable - # "roles/resourcemanager.organizationViewer" = [ - # "domain:${var.organization.domain}" - # ] + for k, v in local.iam_group_bindings : "group:${k}" => { + authoritative = [] + additive = v.additive + } + } + ) + _iam_bindings_auth = flatten([ + for member, data in local._iam_bindings : [ + for role in data.authoritative : { + member = member + role = role + } + ] + ]) + _iam_bindings_add = flatten([ + for member, data in local._iam_bindings : [ + for role in data.additive : { + member = member + role = role + } + ] + ]) + group_iam = { + for k, v in local.iam_group_bindings : k => v.authoritative + } + iam = merge( + { + for r in local.iam_delete_roles : r => [] }, - local.billing_mode == "org" ? { - "roles/billing.admin" = [ - local.groups_iam.gcp-billing-admins, - local.groups_iam.gcp-organization-admins, - module.automation-tf-bootstrap-sa.iam_email, - module.automation-tf-resman-sa.iam_email - ], - "roles/billing.costsManager" = [ - local.groups_iam.gcp-billing-admins, - local.groups_iam.gcp-organization-admins, - module.automation-tf-bootstrap-sa.iam_email, - module.automation-tf-resman-sa.iam_email - ] - } : {} + { + for b in local._iam_bindings_auth : b.role => b.member... + } ) - _iam_bootstrap_user = ( - var.bootstrap_user == null ? [] : ["user:${var.bootstrap_user}"] - ) - iam = { - for role in local.iam_roles : role => distinct(concat( - try(sort(local._iam[role]), []), - try(sort(var.iam[role]), []) - )) + iam_bindings_additive = { + for b in local._iam_bindings_add : "${b.role}-${b.member}" => { + member = b.member + role = b.role + } } - iam_additive = { - for role in local.iam_roles_additive : role => distinct(concat( - try(sort(local._iam_additive[role]), []), - try(sort(var.iam_additive[role]), []) - )) - } - iam_roles = distinct(concat( - keys(local._iam), keys(var.iam) - )) - iam_roles_additive = distinct(concat( - keys(local._iam_additive), keys(var.iam_additive) - )) } module "organization" { @@ -129,40 +69,52 @@ module "organization" { organization_id = "organizations/${var.organization.id}" # human (groups) IAM bindings group_iam = { - (local.groups.gcp-organization-admins) = [ - "roles/cloudasset.owner", - "roles/cloudsupport.admin", - "roles/compute.osAdminLogin", - "roles/compute.osLoginExternalUser", - "roles/owner", - # granted via additive roles - # roles/iam.organizationRoleAdmin - # roles/orgpolicy.policyAdmin - "roles/resourcemanager.folderAdmin", - "roles/resourcemanager.organizationAdmin", - "roles/resourcemanager.projectCreator", - ], - (local.groups.gcp-network-admins) = [ - "roles/cloudasset.owner", - "roles/cloudsupport.techSupportEditor", - ] - (local.groups.gcp-security-admins) = [ - "roles/cloudasset.owner", - "roles/cloudsupport.techSupportEditor", - "roles/iam.securityReviewer", - "roles/logging.admin", - "roles/securitycenter.admin", - ], - (local.groups.gcp-support) = [ - "roles/cloudsupport.techSupportEditor", - "roles/logging.viewer", - "roles/monitoring.viewer", - ] + for k, v in local.group_iam : + k => distinct(concat(v, lookup(var.group_iam, k, []))) } # machine (service accounts) IAM bindings - iam = local.iam + iam = merge( + { + for k, v in local.iam : k => distinct(concat(v, lookup(var.iam, k, []))) + }, + { + for k, v in var.iam : k => v if lookup(local.iam, k, null) == null + } + ) # additive bindings, used for roles co-managed by different stages - iam_additive = local.iam_additive + iam_bindings_additive = merge( + local.iam_bindings_additive, + var.iam_bindings_additive + ) + # delegated role grant for resource manager service account + iam_bindings = { + sa_resman_delegated_iam = { + members = [module.automation-tf-resman-sa.iam_email] + role = module.organization.custom_role_id[var.custom_role_names.organization_iam_admin] + condition = { + expression = format( + "api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly([%s])", + join(",", formatlist("'%s'", concat( + [ + "roles/accesscontextmanager.policyAdmin", + "roles/compute.orgFirewallPolicyAdmin", + "roles/compute.xpnAdmin", + "roles/orgpolicy.policyAdmin", + "roles/resourcemanager.organizationViewer", + module.organization.custom_role_id[var.custom_role_names.tenant_network_admin] + ], + local.billing_mode == "org" ? [ + "roles/billing.admin", + "roles/billing.costsManager", + "roles/billing.user", + ] : [] + ))) + ) + title = "automation_sa_delegated_grants" + description = "Automation service account delegated grants." + } + } + } custom_roles = merge(var.custom_roles, { # this is needed for use in additive IAM bindings, to avoid conflicts (var.custom_role_names.organization_iam_admin) = [ @@ -200,36 +152,3 @@ module "organization" { } } } - -# assign the custom restricted Organization Admin role to the relevant service -# accounts, with a condition that only enables granting specific roles; -# these roles use additive bindings everywhere to avoid conflicts / permadiffs - -resource "google_organization_iam_binding" "org_admin_delegated" { - org_id = var.organization.id - role = module.organization.custom_role_id[var.custom_role_names.organization_iam_admin] - members = [module.automation-tf-resman-sa.iam_email] - condition { - title = "automation_sa_delegated_grants" - description = "Automation service account delegated grants." - expression = format( - "api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly([%s])", - join(",", formatlist("'%s'", concat( - [ - "roles/accesscontextmanager.policyAdmin", - "roles/compute.orgFirewallPolicyAdmin", - "roles/compute.xpnAdmin", - "roles/orgpolicy.policyAdmin", - "roles/resourcemanager.organizationViewer", - module.organization.custom_role_id[var.custom_role_names.tenant_network_admin] - ], - local.billing_mode == "org" ? [ - "roles/billing.admin", - "roles/billing.costsManager", - "roles/billing.user", - ] : [] - ))) - ) - } - depends_on = [module.organization] -} diff --git a/fast/stages/0-bootstrap/variables.tf b/fast/stages/0-bootstrap/variables.tf index 1008c240..93997979 100644 --- a/fast/stages/0-bootstrap/variables.tf +++ b/fast/stages/0-bootstrap/variables.tf @@ -129,6 +129,14 @@ variable "federated_identity_providers" { # } } +variable "group_iam" { + description = "Organization-level authoritative IAM binding for groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + + variable "groups" { # https://cloud.google.com/docs/enterprise/setup-checklist description = "Group names or emails to grant organization-level permissions. If just the name is provided, the default organization domain is assumed." @@ -150,30 +158,35 @@ variable "groups" { variable "iam" { description = "Organization-level custom IAM settings in role => [principal] format." type = map(list(string)) + nullable = false default = {} } -variable "iam_additive" { - description = "Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings." - type = map(list(string)) - default = {} +variable "iam_bindings_additive" { + description = "Organization-level custom additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } variable "locations" { description = "Optional locations for GCS, BigQuery, and logging buckets created here." type = object({ - bq = string - gcs = string - logging = string - pubsub = list(string) + bq = optional(string, "EU") + gcs = optional(string, "EU") + logging = optional(string, "global") + pubsub = optional(list(string), []) }) - default = { - bq = "EU" - gcs = "EU" - logging = "global" - pubsub = [] - } nullable = false + default = {} } # See https://cloud.google.com/architecture/exporting-stackdriver-logging-for-security-and-access-analytics diff --git a/fast/stages/1-resman/README.md b/fast/stages/1-resman/README.md index d938020f..946a767b 100644 --- a/fast/stages/1-resman/README.md +++ b/fast/stages/1-resman/README.md @@ -342,11 +342,11 @@ Due to its simplicity, this stage lends itself easily to customizations: adding | [branch-data-platform.tf](./branch-data-platform.tf) | Data Platform stages resources. | folder · gcs · iam-service-account | google_organization_iam_member | | [branch-gke.tf](./branch-gke.tf) | GKE multitenant stage resources. | folder · gcs · iam-service-account | | | [branch-networking.tf](./branch-networking.tf) | Networking stage resources. | folder · gcs · iam-service-account | | -| [branch-project-factory.tf](./branch-project-factory.tf) | Project factory stage resources. | gcs · iam-service-account | google_organization_iam_member | +| [branch-project-factory.tf](./branch-project-factory.tf) | Project factory stage resources. | gcs · iam-service-account | | | [branch-sandbox.tf](./branch-sandbox.tf) | Sandbox stage resources. | folder · gcs · iam-service-account | google_organization_iam_member | | [branch-security.tf](./branch-security.tf) | Security stage resources. | folder · gcs · iam-service-account | | | [branch-teams.tf](./branch-teams.tf) | Team stage resources. | folder · gcs · iam-service-account | | -| [branch-tenants.tf](./branch-tenants.tf) | Lightweight tenant resources. | folder · gcs · iam-service-account · organization · project | | +| [branch-tenants.tf](./branch-tenants.tf) | Lightweight tenant resources. | folder · gcs · iam-service-account · project | | | [cicd-data-platform.tf](./cicd-data-platform.tf) | CI/CD resources for the data platform branch. | iam-service-account · source-repository | | | [cicd-gke.tf](./cicd-gke.tf) | CI/CD resources for the data platform branch. | iam-service-account · source-repository | | | [cicd-networking.tf](./cicd-networking.tf) | CI/CD resources for the networking branch. | iam-service-account · source-repository | | @@ -354,6 +354,7 @@ Due to its simplicity, this stage lends itself easily to customizations: adding | [cicd-security.tf](./cicd-security.tf) | CI/CD resources for the security branch. | iam-service-account · source-repository | | | [cicd-teams.tf](./cicd-teams.tf) | CI/CD resources for individual teams. | iam-service-account · source-repository | | | [main.tf](./main.tf) | Module-level locals and resources. | | | +| [organization-iam.tf](./organization-iam.tf) | Organization-level IAM bindings locals. | | | | [organization.tf](./organization.tf) | Organization policies. | organization | | | [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file | | [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | | google_storage_bucket_object | diff --git a/fast/stages/1-resman/branch-project-factory.tf b/fast/stages/1-resman/branch-project-factory.tf index 8ae9b4fe..7f2fbf28 100644 --- a/fast/stages/1-resman/branch-project-factory.tf +++ b/fast/stages/1-resman/branch-project-factory.tf @@ -79,33 +79,3 @@ module "branch-pf-prod-gcs" { "roles/storage.objectAdmin" = [module.branch-pf-prod-sa.0.iam_email] } } - -resource "google_organization_iam_member" "org_policy_admin_pf_dev" { - count = var.fast_features.project_factory ? 1 : 0 - org_id = var.organization.id - role = "roles/orgpolicy.policyAdmin" - member = module.branch-pf-dev-sa.0.iam_email - condition { - title = "org_policy_tag_pf_scoped_dev" - description = "Org policy tag scoped grant for project factory dev." - expression = <<-END - resource.matchTag('${var.organization.id}/${var.tag_names.context}', 'teams') - && - resource.matchTag('${var.organization.id}/${var.tag_names.environment}', 'development') - END - } -} - -resource "google_organization_iam_member" "org_policy_admin_pf_prod" { - count = var.fast_features.project_factory ? 1 : 0 - org_id = var.organization.id - role = "roles/orgpolicy.policyAdmin" - member = module.branch-pf-prod-sa.0.iam_email - condition { - title = "org_policy_tag_pf_scoped_prod" - description = "Org policy tag scoped grant for project factory prod." - expression = <<-END - resource.matchTag('${var.organization.id}/${var.tag_names.context}', 'teams') - END - } -} diff --git a/fast/stages/1-resman/branch-tenants.tf b/fast/stages/1-resman/branch-tenants.tf index 8a391a07..c5a70fe9 100644 --- a/fast/stages/1-resman/branch-tenants.tf +++ b/fast/stages/1-resman/branch-tenants.tf @@ -23,27 +23,6 @@ locals { module.tenant-self-iac-sa[k].iam_email ] } - tenant_org_iam = compact(flatten([ - for k, v in var.tenants : [ - "group:${v.admin_group_email}", - v.organization != null ? "domain:${v.organization.domain}" : null - ] - ])) -} - - -# org-level roles for each tenant (additive) - -module "tenant-org-iam" { - source = "../../../modules/organization" - organization_id = "organizations/${var.organization.id}" - iam_additive = { - "roles/compute.osLoginExternalUser" = [ - for k, v in var.tenants : - "domain:${v.organization.domain}" if v.organization != null - ] - "roles/resourcemanager.organizationViewer" = local.tenant_org_iam - } } # top-level "Tenants" folder @@ -107,7 +86,9 @@ module "tenant-core-folder-iam" { folder_create = false iam = merge( { - "roles/owner" = [module.tenant-core-sa[each.key].iam_email] + "roles/owner" = [ + module.tenant-core-sa[each.key].iam_email + ] "roles/viewer" = local.tenant_iam[each.key] }, { diff --git a/fast/stages/1-resman/organization-iam.tf b/fast/stages/1-resman/organization-iam.tf new file mode 100644 index 00000000..28764a25 --- /dev/null +++ b/fast/stages/1-resman/organization-iam.tf @@ -0,0 +1,141 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Organization-level IAM bindings locals. + +locals { + iam_bindings_additive = merge( + # network and security + { + sa_net_fw_policy_admin = { + member = module.branch-network-sa.iam_email + role = "roles/compute.orgFirewallPolicyAdmin" + } + sa_net_xpn_admin = { + member = module.branch-network-sa.iam_email + role = "roles/compute.xpnAdmin" + } + sa_sec_vpcsc_admin = { + member = module.branch-security-sa.iam_email + role = "roles/accesscontextmanager.policyAdmin" + } + }, + # optional billing roles for network and security + local.billing_mode != "org" ? {} : { + sa_net_billing = { + member = module.branch-network-sa.iam_email + role = "roles/billing.user" + } + sa_sec_billing = { + member = module.branch-security-sa.iam_email + role = "roles/billing.user" + } + }, + # optional billing roles for data platform + local.billing_mode != "org" || !var.fast_features.data_platform ? {} : { + sa_dp_dev_billing = { + member = module.branch-dp-dev-sa.0.iam_email + role = "roles/billing.user" + } + sa_dp_prod_billing = { + member = module.branch-dp-prod-sa.0.iam_email + role = "roles/billing.user" + } + }, + # optional billing roles for GKE + local.billing_mode != "org" || !var.fast_features.gke ? {} : { + sa_gke_dev_billing = { + member = module.branch-gke-dev-sa.0.iam_email + role = "roles/billing.user" + } + sa_gke_prod_billing = { + member = module.branch-gke-prod-sa.0.iam_email + role = "roles/billing.user" + } + }, + # optional billing roles for project factory + local.billing_mode != "org" || !var.fast_features.project_factory ? {} : { + sa_pf_dev_billing = { + member = module.branch-pf-dev-sa.0.iam_email + role = "roles/billing.user" + } + sa_pf_dev_costs_manager = { + member = module.branch-pf-dev-sa.0.iam_email + role = "roles/billing.costsManager" + } + sa_pf_prod_billing = { + member = module.branch-pf-prod-sa.0.iam_email + role = "roles/billing.user" + } + sa_pf_prod_costs_manager = { + member = module.branch-pf-prod-sa.0.iam_email + role = "roles/billing.costsManager" + } + }, + # scoped org policy admin grants for project factory + !var.fast_features.project_factory ? {} : { + sa_pf_dev_conditional_org_policy = { + member = module.branch-pf-dev-sa.0.iam_email + role = "roles/orgpolicy.policyAdmin" + condition = { + title = "org_policy_tag_pf_scoped_dev" + description = "Org policy tag scoped grant for project factory dev." + expression = <<-END + resource.matchTag('${var.organization.id}/${var.tag_names.context}', 'teams') + && + resource.matchTag('${var.organization.id}/${var.tag_names.environment}', 'development') + END + } + } + sa_pf_prod_conditional_org_policy = { + member = module.branch-pf-prod-sa.0.iam_email + role = "roles/orgpolicy.policyAdmin" + condition = { + title = "org_policy_tag_pf_scoped_prod" + description = "Org policy tag scoped grant for project factory prod." + expression = <<-END + resource.matchTag('${var.organization.id}/${var.tag_names.context}', 'teams') + END + } + } + }, + # lightweight tenant roles + { + for k, v in var.tenants : "oslogin_ext_user-tenant_${k}" => { + member = "domain:${v.organization.domain}" + role = "roles/compute.osLoginExternalUser" + } if v.organization != null + }, + { + for k, v in var.tenants : "org-viewer-tenant_${k}_domain" => { + member = "domain:${v.organization.domain}" + role = "roles/resourcemanager.organizationViewer" + } if v.organization != null + }, + { + for k, v in var.tenants : "org-viewer-tenant_${k}_admin" => { + member = "group:${v.admin_group_email}" + role = "roles/resourcemanager.organizationViewer" + } + }, + local.billing_mode != "org" ? {} : { + for k, v in var.tenants : "billing_user-tenant_${k}_billing_admin" => { + member = "group:${v.admin_group_email}" + role = "roles/billing.user" + } + }, + ) +} diff --git a/fast/stages/1-resman/organization.tf b/fast/stages/1-resman/organization.tf index 033fe8a3..c6860ad6 100644 --- a/fast/stages/1-resman/organization.tf +++ b/fast/stages/1-resman/organization.tf @@ -49,39 +49,8 @@ locals { module "organization" { source = "../../../modules/organization" organization_id = "organizations/${var.organization.id}" - # IAM additive bindings, granted via the restricted Organization Admin custom - # role assigned in stage 00; they need to be additive to avoid conflicts - iam_additive = merge( - { - "roles/accesscontextmanager.policyAdmin" = [ - module.branch-security-sa.iam_email - ] - "roles/compute.orgFirewallPolicyAdmin" = [ - module.branch-network-sa.iam_email - ] - "roles/compute.xpnAdmin" = [ - module.branch-network-sa.iam_email - ] - }, - local.billing_mode == "org" ? { - "roles/billing.costsManager" = concat( - local.branch_optional_sa_lists.pf-dev, - local.branch_optional_sa_lists.pf-prod - ) - "roles/billing.user" = concat( - [ - module.branch-network-sa.iam_email, - module.branch-security-sa.iam_email, - ], - local.branch_optional_sa_lists.dp-dev, - local.branch_optional_sa_lists.dp-prod, - local.branch_optional_sa_lists.gke-dev, - local.branch_optional_sa_lists.gke-prod, - local.branch_optional_sa_lists.pf-dev, - local.branch_optional_sa_lists.pf-prod, - ) - } : {} - ) + # additive bindings via delegated IAM grant set in stage 0 + iam_bindings_additive = local.iam_bindings_additive # sample subset of useful organization policies, edit to suit requirements org_policies = { "iam.allowedPolicyMemberDomains" = { @@ -116,53 +85,48 @@ module "organization" { org_policies_data_path = "${var.data_dir}/org-policies" # do not assign tagViewer or tagUser roles here on tag keys and values as # they are managed authoritatively and will break multitenant stages - tags = merge( - local.tags, - { - (var.tag_names.context) = { - description = "Resource management context." - iam = {} - values = { - data = null - gke = null - networking = null - sandbox = null - security = null - teams = null - tenant = null - } + tags = merge(local.tags, { + (var.tag_names.context) = { + description = "Resource management context." + iam = {} + values = { + data = null + gke = null + networking = null + sandbox = null + security = null + teams = null + tenant = null } - (var.tag_names.environment) = { - description = "Environment definition." - iam = {} - values = { - development = null - production = null - } + } + (var.tag_names.environment) = { + description = "Environment definition." + iam = {} + values = { + development = null + production = null } - (var.tag_names.org-policies) = { - description = "Organization policy conditions." - iam = {} - values = { - allowed-policy-member-domains-all = merge({}, try( - local.tags[var.tag_names.org-policies].values.allowed-policy-member-domains-all, - {} - )) - } + } + (var.tag_names.org-policies) = { + description = "Organization policy conditions." + iam = {} + values = { + allowed-policy-member-domains-all = merge({}, try( + local.tags[var.tag_names.org-policies].values.allowed-policy-member-domains-all, + {} + )) } - (var.tag_names.tenant) = { - description = "Organization tenant." - values = { - for k, v in var.tenants : k => { - description = v.descriptive_name - iam = { - "roles/resourcemanager.tagViewer" = local.tenant_iam[k] - } + } + (var.tag_names.tenant) = { + description = "Organization tenant." + values = { + for k, v in var.tenants : k => { + description = v.descriptive_name + iam = { + "roles/resourcemanager.tagViewer" = local.tenant_iam[k] } } } } - ) + }) } - -# organization policy conditional roles are in the relevant branch files diff --git a/fast/stages/2-security/core-dev.tf b/fast/stages/2-security/core-dev.tf index 856047b0..1b494947 100644 --- a/fast/stages/2-security/core-dev.tf +++ b/fast/stages/2-security/core-dev.tf @@ -46,7 +46,7 @@ module "dev-sec-kms" { name = "dev-${each.key}" } # rename to `key_iam` to switch to authoritative bindings - key_iam_additive = { + key_iam = { for k, v in local.kms_locations_keys[each.key] : k => v.iam } keys = local.kms_locations_keys[each.key] diff --git a/fast/stages/2-security/core-prod.tf b/fast/stages/2-security/core-prod.tf index ecbce18f..559ff32f 100644 --- a/fast/stages/2-security/core-prod.tf +++ b/fast/stages/2-security/core-prod.tf @@ -45,7 +45,7 @@ module "prod-sec-kms" { name = "prod-${each.key}" } # rename to `key_iam` to switch to authoritative bindings - key_iam_additive = { + key_iam = { for k, v in local.kms_locations_keys[each.key] : k => v.iam } keys = local.kms_locations_keys[each.key] diff --git a/fast/stages/3-project-factory/dev/README.md b/fast/stages/3-project-factory/dev/README.md index b9eb9c76..2073e759 100644 --- a/fast/stages/3-project-factory/dev/README.md +++ b/fast/stages/3-project-factory/dev/README.md @@ -13,43 +13,13 @@ A single factory creates projects in a well-defined context, according to your r Projects for each environment across different teams are created by dedicated service accounts, as exemplified in the diagram above. While there's no intrinsic limitation regarding where the project factory can create a projects, the IAM bindings for the service account effectively enforce boundaries (e.g., the production service account shouldn't be able to create or have any access to the development projects, and vice versa). -The project factory takes care of the following activities: - -- Project creation -- API/Services enablement -- Service accounts creation -- IAM roles assignment for groups and service accounts -- KMS keys roles assignment -- Shared VPC attachment and subnets IAM binding -- DNS zones creation and visibility configuration -- Project-level org policies definition -- Billing setup (billing account attachment and budget configuration) -- Essential contacts definition (for [budget alerts](https://cloud.google.com/billing/docs/how-to/budgets) and [important notifications](https://cloud.google.com/resource-manager/docs/managing-notification-contacts?hl=en)) +The project factory exposes all the features of the underlying [project module](../../../../modules/project/), including Shared VPC service project attachment, VPC SC perimeter membership, etc. ## How to run this stage This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../0-bootstrap), [`01-resman`](../../1-resman), 02-networking (either [VPN](../../2-networking-b-vpn), [NVA](../../2-networking-c-nva), [NVA with BGP support](../../2-networking-e-nva-bgp)) and [`02-security`](../../2-security)) have been run. -It's of course possible to run this stage in isolation, by making sure the architectural prerequisites are satisfied (e.g., networking), and that the Service Account running the stage is granted the roles/permissions below: - -- One service account per environment, each with appropriate permissions - - at the organization level a custom role for networking operations including the following permissions - - `"compute.organizations.enableXpnResource"`, - - `"compute.organizations.disableXpnResource"`, - - `"compute.subnetworks.setIamPolicy"`, - - `"dns.networks.bindPrivateDNSZone"` - - and role `"roles/orgpolicy.policyAdmin"` - - on each folder where projects are created - - `"roles/logging.admin"` - - `"roles/owner"` - - `"roles/resourcemanager.folderAdmin"` - - `"roles/resourcemanager.projectCreator"` - - on the host project for the Shared VPC - - `"roles/browser"` - - `"roles/compute.viewer"` - - `"roles/dns.admin"` -- If networking is used (e.g., for VMs, GKE Clusters or AppEngine flex), VPC Host projects and their subnets should exist when creating projects -- If per-environment DNS sub-zones are required, one "root" zone per environment should exist when creating projects (e.g., dev.gcp.example.com.) +It's of course possible to run this stage in isolation, by making sure the architectural prerequisites are satisfied (e.g., networking), and that the Service Account running the stage is granted the appropriate roles. ### Provider and Terraform variables @@ -83,14 +53,11 @@ gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/2-networking.aut gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/2-security.auto.tfvars.json ./ ``` -If you're not using Fast, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning. +If you're not using FAST, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning. -Besides the values above, a project factory takes 2 additional inputs: +Besides the values above, the project factory is drive by data files, with one file per project. -- `data/defaults.yaml`, manually configured by adapting the [`data/defaults.yaml`](./data/defaults.yaml), which defines per-environment default values e.g., for billing alerts and labels. -- `data/projects/*.yaml`, one file per project (optionally grouped in folders), which configures each project. A [`data/projects/project.yaml`](./data/projects/project.yaml.sample) is provided as reference and documentation for the schema. Projects will be named after the filename, e.g., `fast-dev-lab0.yaml` will create project `fast-dev-lab0`. - -Once the configuration is complete, run the project factory by running +Once the configuration is complete, run the project factory with: ```bash terraform init @@ -99,7 +66,6 @@ terraform apply - ## Files | name | description | modules | @@ -113,17 +79,13 @@ terraform apply | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [billing_account](variables.tf#L19) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | -| [prefix](variables.tf#L60) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | -| [data_dir](variables.tf#L32) | Relative path for the folder storing configuration data. | string | | "data/projects" | | -| [defaults_file](variables.tf#L38) | Relative path for the file storing the project factory configuration. | string | | "data/defaults.yaml" | | -| [environment_dns_zone](variables.tf#L44) | DNS zone suffix for environment. | string | | null | 2-networking | -| [host_project_ids](variables.tf#L51) | Host project for the shared VPC. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L71) | Self link for the shared VPC. | object({…}) | | null | 2-networking | +| [factory_data](variables.tf#L32) | Project data from either YAML files or externally parsed data. | object({…}) | ✓ | | | +| [prefix](variables.tf#L48) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [projects](outputs.tf#L17) | Created projects and service accounts. | | | - +| [projects](outputs.tf#L17) | Created projects. | | | +| [service_accounts](outputs.tf#L22) | Created service accounts. | | | diff --git a/fast/stages/3-project-factory/dev/main.tf b/fast/stages/3-project-factory/dev/main.tf index e38348fe..261351ca 100644 --- a/fast/stages/3-project-factory/dev/main.tf +++ b/fast/stages/3-project-factory/dev/main.tf @@ -16,43 +16,24 @@ # tfdoc:file:description Project factory. - -locals { - _defaults = yamldecode(file(var.defaults_file)) - _defaults_net = { - billing_account_id = var.billing_account.id - environment_dns_zone = var.environment_dns_zone - shared_vpc_self_link = try(var.vpc_self_links["dev-spoke-0"], null) - vpc_host_project = try(var.host_project_ids["dev-spoke-0"], null) - } - defaults = merge(local._defaults, local._defaults_net) - projects = { - for f in fileset("${var.data_dir}", "**/*.yaml") : - trimsuffix(f, ".yaml") => yamldecode(file("${var.data_dir}/${f}")) - } -} - module "projects" { - source = "../../../../blueprints/factories/project-factory" - for_each = local.projects - defaults = local.defaults - project_id = each.key - billing_account_id = try(each.value.billing_account_id, null) - billing_alert = try(each.value.billing_alert, null) - dns_zones = try(each.value.dns_zones, []) - essential_contacts = try(each.value.essential_contacts, []) - folder_id = try(each.value.folder_id, local.defaults.folder_id) - group_iam = try(each.value.group_iam, {}) - iam = try(each.value.iam, {}) - kms_service_agents = try(each.value.kms_service_agents, {}) - labels = try(each.value.labels, {}) - org_policies = try(each.value.org_policies, null) - prefix = var.prefix - service_accounts = try(each.value.service_accounts, {}) - service_accounts_iam = try(each.value.service_accounts_iam, {}) - services = try(each.value.services, []) - service_identities_iam = try(each.value.service_identities_iam, {}) - vpc = try(each.value.vpc, null) + source = "../../../../blueprints/factories/project-factory" + data_defaults = { + billing_account = var.billing_account.id + # more defaults are available, check the project factory variables + } + data_merges = { + labels = { + environment = "dev" + } + services = [ + "stackdriver.googleapis.com" + ] + } + data_overrides = { + prefix = var.prefix + } + factory_data = var.factory_data } diff --git a/fast/stages/3-project-factory/dev/outputs.tf b/fast/stages/3-project-factory/dev/outputs.tf index 59ecff95..2c86ac9c 100644 --- a/fast/stages/3-project-factory/dev/outputs.tf +++ b/fast/stages/3-project-factory/dev/outputs.tf @@ -15,6 +15,11 @@ */ output "projects" { - description = "Created projects and service accounts." - value = module.projects + description = "Created projects." + value = module.projects.projects +} + +output "service_accounts" { + description = "Created service accounts." + value = module.projects.service_accounts } diff --git a/fast/stages/3-project-factory/dev/variables.tf b/fast/stages/3-project-factory/dev/variables.tf index 5ad49f77..d004aeb8 100644 --- a/fast/stages/3-project-factory/dev/variables.tf +++ b/fast/stages/3-project-factory/dev/variables.tf @@ -29,32 +29,20 @@ variable "billing_account" { } } -variable "data_dir" { - description = "Relative path for the folder storing configuration data." - type = string - default = "data/projects" -} - -variable "defaults_file" { - description = "Relative path for the file storing the project factory configuration." - type = string - default = "data/defaults.yaml" -} - -variable "environment_dns_zone" { - # tfdoc:variable:source 2-networking - description = "DNS zone suffix for environment." - type = string - default = null -} - -variable "host_project_ids" { - # tfdoc:variable:source 2-networking - description = "Host project for the shared VPC." +variable "factory_data" { + description = "Project data from either YAML files or externally parsed data." type = object({ - dev-spoke-0 = string + data = optional(map(any)) + data_path = optional(string) }) - default = null + nullable = false + validation { + condition = ( + (var.factory_data.data != null ? 1 : 0) + + (var.factory_data.data_path != null ? 1 : 0) + ) == 1 + error_message = "One of data or data_path needs to be set." + } } variable "prefix" { @@ -67,12 +55,3 @@ variable "prefix" { error_message = "Use a maximum of 9 characters for prefix." } } - -variable "vpc_self_links" { - # tfdoc:variable:source 2-networking - description = "Self link for the shared VPC." - type = object({ - dev-spoke-0 = string - }) - default = null -} diff --git a/modules/__docs/20230816-iam-refactor.md b/modules/__docs/20230816-iam-refactor.md new file mode 100644 index 00000000..a252d27a --- /dev/null +++ b/modules/__docs/20230816-iam-refactor.md @@ -0,0 +1,148 @@ +# Refactor IAM interface + +**authors:** [Ludo](https://github.com/ludoo), [Julio](https://github.com/juliocc) +**last modified:** August 17, 2023 + +## Status + +Discussed. + +## Context + +Our modules IAM interface has evolved organically to progressively support more functionality, resulting in a large variable surface, lack of support for some key features like conditions, and some fragility for specific use cases. + +We currently support, with uneven coverage across modules: + +- authoritative `iam` in `ROLE => [PRINCIPALS]` format +- authoritative `group_iam` in `GROUP => [ROLES]` format +- legacy additive `iam_additive` in `ROLE => [PRINCIPALS]` format which breaks for dynamic values +- legacy additive `iam_additive_members` in `PRINCIPAL => [ROLES]` format which breaks for dynamic values +- new additive `iam_members` in `KEY => {role: ROLE, member: MEMBER, condition: CONDITION}` format which works with dynamic values and supports conditions +- policy authoritative `iam_policy` +- specific support for third party resource bindings in the service account module + +## Proposal + +### Authoritative bindings + +These tend to work well in practice, and the current `iam` and `group_iam` variables are simple to use with good coverage across modules. + +The only small use case that they do not cover is IAM conditions, which are easy to implement but would render the interface more verbose for the majority of cases where conditions are not needed. + +The **proposal** for authoritative bindings is to + +- leave the current interface in place (`iam` and `group_iam`) +- expand coverage so that all modules who have iam resources expose both +- add a new `iam_bindings` variable to support authoritative IAM with conditions + +The new `iam_bindings` variable will look like this: + +```hcl +variable "iam_bindings" { + description = "Authoritative IAM bindings with support for conditions, in {ROLE => { members = [], condition = {}}} format." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) +} +``` + +This variable will not be internally merged in modules with `iam` or `group_iam`. + +### Additive bindings + +Additive bindings have evolved to mimick authoritative ones, but the result is an interface which is bloated (no one uses `iam_additive_members`), and hard to understand and use without triggering dynamic errors. Coverage is also spotty and uneven across modules, and the interface needs to support aliasing of project service accounts in the project module to work around dynamic errors. + +The `iam_additive` variable is used in a special patterns in data blueprints, to allow code to not mess up existing IAM bindings in an external project on destroy. This pattern only works in a limited set of cases, where principals are passed in via static variables or refer to "magic" static outputs in our modules. This is a simple example of the pattern: + +```hcl +locals { + iam = { + "roles/viewer" = [ + module.sa.iam_email, + var.group.admins + ] + } +} +module "project" { + iam = ( + var.project_create == null ? {} : local.iam + ) + iam_additive = ( + var.project_create != null ? {} : local.iam + ) +} +``` + +The **proposal** for authoritative bindings is to + +- remove `iam_additive` and `iam_additive_members` from the interface +- add a new `iam_bindings_additive` variable + +Once new variables are in place, migrate existing blueprints to using `iam_bindings_additive` using one of the two available patterns: + +- the flat verbose one where bindings are declared in the module call +- the more complex one that moves roles out to `locals` and uses them in `for` loops + +The new variable will closely follow the type of the authoritative `iam_bindings` variable described above: + +```hcl +variable "iam_bindings_additive" { + description = "Additive IAM bindings with support for conditions, in {KEY => { role = ROLE, members = [], condition = {}}} format." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) +} +``` + +### IAM policy + +The **proposal** is to remove the IAM policy variable and resources, as its coverage is very uneven and we never used it in practice. This will also simplify data access log management, which is currently split between its own variable/resource and the IAM policy ones. + +## Decision + +The proposal above summarizes the state of discussions between the authors, and implementation will be tested. + +## Consequences + +A few data blueprints that leverage `iam_additive` will need to be refactored to use the new variable, using one of the following patterns: + +```hcl +locals { + network_sa_roles = [ + "roles/compute.orgFirewallPolicyAdmin", + "roles/compute.xpnAdmin" + ] +} + +module "organization" { + source = "../../../modules/organization" + organization_id = "organizations/${var.organization.id}" + iam_bindings_additive = merge( + # IAM bindings via locals pattern + { + for r in local.network_sa_roles : "network_sa-${r}" : { + member = module.branch-network-sa.iam_email + role = r + } + }, + # IAM bindings via explicit reference pattern + { + security_sa = { + member = module.branch-security-sa.iam_email + role = "roles/accesscontextmanager.policyAdmin" + } + } + ) +} +``` diff --git a/modules/__docs/README.md b/modules/__docs/README.md new file mode 100644 index 00000000..d6c722c2 --- /dev/null +++ b/modules/__docs/README.md @@ -0,0 +1,3 @@ +# FAST architectural documents + +This folder contains assorted bits of documentation used to log current architectural choices, or past decisions. Format is inspired by [Michael Nygard's decision record template](https://github.com/joelparkerhenderson/architecture-decision-record/blob/main/templates/decision-record-template-by-michael-nygard/index.md). diff --git a/modules/cloud-run/README.md b/modules/cloud-run/README.md index 803ddcb6..7be58bf3 100644 --- a/modules/cloud-run/README.md +++ b/modules/cloud-run/README.md @@ -251,6 +251,7 @@ module "cloud_run" { By default `Compute default service account` is used to trigger Cloud Run. If you want to use custom Service Account you can either provide your own in `eventarc_triggers.service_account_email` or set `eventarc_triggers.service_account_create` to true and service account named `tf-cr-trigger-${var.name}` will be created with `roles/run.invoker` granted on this Cloud Run service. Example using provided service account: + ```hcl module "cloud_run" { source = "./fabric/modules/cloud-run" @@ -275,6 +276,7 @@ module "cloud_run" { ``` Example using automatically created service account: + ```hcl module "cloud_run" { source = "./fabric/modules/cloud-run" @@ -296,7 +298,6 @@ module "cloud_run" { # tftest modules=1 resources=5 inventory=trigger-service-account.yaml ``` - ### Service account To use a custom service account managed by the module, set `service_account_create` to `true` and leave `service_account` set to `null` value (default). diff --git a/modules/data-catalog-policy-tag/README.md b/modules/data-catalog-policy-tag/README.md index 5a1c2055..b08a9feb 100644 --- a/modules/data-catalog-policy-tag/README.md +++ b/modules/data-catalog-policy-tag/README.md @@ -5,6 +5,7 @@ This module simplifies the creation of [Data Catalog](https://cloud.google.com/d Note: Data Catalog is still in beta, hence this module currently uses the beta provider. +- [IAM](#iam) - [Examples](#examples) - [Simple Taxonomy with policy tags](#simple-taxonomy-with-policy-tags) - [Taxonomy with IAM binding](#taxonomy-with-iam-binding) @@ -13,6 +14,18 @@ Note: Data Catalog is still in beta, hence this module currently uses the beta p - [TODO](#todo) +## IAM + +IAM is managed via several variables that implement different features and levels of control: + +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions + +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. + +Refer to the [project module](../project/README.md#iam) for examples of the IAM interface. + ## Examples ### Simple Taxonomy with policy tags @@ -52,7 +65,7 @@ module "cmn-dc" { iam = { "roles/datacatalog.categoryAdmin" = ["group:GROUP_NAME@example.com"] } - iam_members = { + iam_bindings_additive = { am1-admin = { member = "user:am1@example.com" role = "roles/datacatalog.categoryAdmin" @@ -66,18 +79,17 @@ module "cmn-dc" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L74) | Name of this taxonomy. | string | ✓ | | -| [project_id](variables.tf#L89) | GCP project id. | | ✓ | | +| [name](variables.tf#L76) | Name of this taxonomy. | string | ✓ | | +| [project_id](variables.tf#L91) | GCP project id. | | ✓ | | | [activated_policy_types](variables.tf#L17) | A list of policy types that are activated for this taxonomy. | list(string) | | ["FINE_GRAINED_ACCESS_CONTROL"] | | [description](variables.tf#L23) | Description of this taxonomy. | string | | "Taxonomy - Terraform managed" | | [group_iam](variables.tf#L29) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L35) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L41) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive_members](variables.tf#L47) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L53) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [location](variables.tf#L68) | Data Catalog Taxonomy location. | string | | "eu" | -| [prefix](variables.tf#L79) | Optional prefix used to generate project id and name. | string | | null | -| [tags](variables.tf#L93) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | +| [iam_bindings](variables.tf#L41) | Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L55) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [location](variables.tf#L70) | Data Catalog Taxonomy location. | string | | "eu" | +| [prefix](variables.tf#L81) | Optional prefix used to generate project id and name. | string | | null | +| [tags](variables.tf#L95) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | ## Outputs diff --git a/modules/data-catalog-policy-tag/iam.tf b/modules/data-catalog-policy-tag/iam.tf index 9e7f3a09..268c0c58 100644 --- a/modules/data-catalog-policy-tag/iam.tf +++ b/modules/data-catalog-policy-tag/iam.tf @@ -23,16 +23,6 @@ locals { ] } _group_iam_roles = distinct(flatten(values(var.group_iam))) - _iam_additive_member_pairs = flatten([ - for member, roles in var.iam_additive_members : [ - for role in roles : { role = role, member = member } - ] - ]) - _iam_additive_pairs = flatten([ - for role, members in var.iam_additive : [ - for member in members : { role = role, member = member } - ] - ]) iam = { for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : role => concat( @@ -40,10 +30,6 @@ locals { try(local._group_iam[role], []) ) } - iam_additive = { - for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) : - "${pair.role}-${pair.member}" => pair - } tags_iam = flatten([ for k, v in var.tags : [ for role, members in v.iam : { @@ -63,20 +49,25 @@ resource "google_data_catalog_taxonomy_iam_binding" "authoritative" { members = each.value } -resource "google_data_catalog_taxonomy_iam_member" "additive" { +resource "google_data_catalog_taxonomy_iam_binding" "bindings" { provider = google-beta - for_each = ( - length(var.iam_additive) + length(var.iam_additive_members) > 0 - ? local.iam_additive - : {} - ) + for_each = var.iam_bindings taxonomy = google_data_catalog_taxonomy.default.id - role = each.value.role - member = each.value.member + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } -resource "google_data_catalog_taxonomy_iam_member" "members" { - for_each = var.iam_members +resource "google_data_catalog_taxonomy_iam_member" "bindings" { + provider = google-beta + for_each = var.iam_bindings_additive taxonomy = google_data_catalog_taxonomy.default.id role = each.value.role member = each.value.member diff --git a/modules/data-catalog-policy-tag/variables.tf b/modules/data-catalog-policy-tag/variables.tf index 2b799a96..b0df313d 100644 --- a/modules/data-catalog-policy-tag/variables.tf +++ b/modules/data-catalog-policy-tag/variables.tf @@ -38,20 +38,22 @@ variable "iam" { default = {} } -variable "iam_additive" { - description = "IAM additive bindings in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} +variable "iam_bindings" { + description = "Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_additive_members" { - description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values." - type = map(list(string)) - default = {} -} - -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string diff --git a/modules/dataplex-datascan/README.md b/modules/dataplex-datascan/README.md index ca8afca7..1c950184 100644 --- a/modules/dataplex-datascan/README.md +++ b/modules/dataplex-datascan/README.md @@ -316,8 +316,8 @@ The input variable 'data' is required to create a DataScan. This value is immuta The input variable 'data' should be an object containing a single key-value pair that can be one of: -* `entity`: The Dataplex entity that represents the data source (e.g. BigQuery table) for DataScan, of the form: `projects/{project_number}/locations/{locationId}/lakes/{lakeId}/zones/{zoneId}/entities/{entityId}`. -* `resource`: The service-qualified full resource name of the cloud resource for a DataScan job to scan against. The field could be: BigQuery table of type "TABLE" for DataProfileScan/DataQualityScan format, e.g: `//bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID`. +- `entity`: The Dataplex entity that represents the data source (e.g. BigQuery table) for DataScan, of the form: `projects/{project_number}/locations/{locationId}/lakes/{lakeId}/zones/{zoneId}/entities/{entityId}`. +- `resource`: The service-qualified full resource name of the cloud resource for a DataScan job to scan against. The field could be: BigQuery table of type "TABLE" for DataProfileScan/DataQualityScan format, e.g: `//bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID`. The example below shows how to specify the data source for DataScan of type `resource`: @@ -380,17 +380,15 @@ module "dataplex-datascan" { ## IAM -IAM is managed via several variables that implement different levels of control: +IAM is managed via several variables that implement different features and levels of control: -* `group_iam` and `iam` configure authoritative bindings that manage individual roles exclusively, mapping to the [`google_project_iam_binding`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_binding) resource -* `iam_additive`, `iam_additive_members` and `iam_members` configure additive bindings that only manage individual role/member pairs, mapping to the [`google_project_iam_member`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_member) resource -* `iam_policy` which controls the entire IAM policy for the project, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions -The authoritative and additive approaches can be used together, provided different roles are managed by each. The IAM policy is incompatible with the other approaches, and must be used with extreme care. +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. -Some care must also be taken with the `group_iam` and `iam_additive_*` variables to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. For additive roles `iam_members` ensures that no dynamic values are used in the internal loop. - -An example is provided below for using some of these variables. +An example is provided below for using some of these variables. Refer to the [project module](../project/README.md#iam) for complete examples of the IAM interface. ```hcl module "dataplex-datascan" { @@ -416,7 +414,7 @@ module "dataplex-datascan" { "roles/dataplex.dataScanViewer" ] } - iam_members = { + iam_bindings_additive = { am1-viewer = { member = "user:am1@example.com" role = "roles/dataplex.dataScanViewer" @@ -433,9 +431,9 @@ module "dataplex-datascan" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [data](variables.tf#L17) | The data source for DataScan. The source can be either a Dataplex `entity` or a BigQuery `resource`. | object({…}) | ✓ | | -| [name](variables.tf#L161) | Name of Dataplex Scan. | string | ✓ | | -| [project_id](variables.tf#L172) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | -| [region](variables.tf#L177) | Region for the Dataplex DataScan. | string | ✓ | | +| [name](variables.tf#L156) | Name of Dataplex Scan. | string | ✓ | | +| [project_id](variables.tf#L167) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | +| [region](variables.tf#L172) | Region for the Dataplex DataScan. | string | ✓ | | | [data_profile_spec](variables.tf#L29) | DataProfileScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec. | object({…}) | | null | | [data_quality_spec](variables.tf#L38) | DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | | [data_quality_spec_file](variables.tf#L80) | Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | @@ -443,13 +441,11 @@ module "dataplex-datascan" { | [execution_schedule](variables.tf#L94) | Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API. | string | | null | | [group_iam](variables.tf#L100) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L107) | Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L114) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive_members](variables.tf#L121) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L127) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_policy](variables.tf#L142) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [incremental_field](variables.tf#L148) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | -| [labels](variables.tf#L154) | Resource labels. | map(string) | | {} | -| [prefix](variables.tf#L166) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | +| [iam_bindings](variables.tf#L114) | Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L128) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [incremental_field](variables.tf#L143) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | +| [labels](variables.tf#L149) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L161) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | ## Outputs diff --git a/modules/dataplex-datascan/iam.tf b/modules/dataplex-datascan/iam.tf index 2a6c4c78..9a496ff1 100644 --- a/modules/dataplex-datascan/iam.tf +++ b/modules/dataplex-datascan/iam.tf @@ -21,16 +21,6 @@ locals { for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null ] } - _iam_additive_pairs = flatten([ - for role, members in var.iam_additive : [ - for member in members : { role = role, member = member } - ] - ]) - _iam_additive_member_pairs = flatten([ - for member, roles in var.iam_additive_members : [ - for role in roles : { role = role, member = member } - ] - ]) iam = { for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : role => concat( @@ -38,13 +28,6 @@ locals { try(local._group_iam[role], []) ) } - iam_additive = { - for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) : - "${pair.role}-${pair.member}" => { - role = pair.role - member = pair.member - } - } } resource "google_dataplex_datascan_iam_binding" "authoritative_for_role" { @@ -56,21 +39,25 @@ resource "google_dataplex_datascan_iam_binding" "authoritative_for_role" { members = each.value } -resource "google_dataplex_datascan_iam_member" "additive" { - for_each = ( - length(var.iam_additive) + length(var.iam_additive_members) > 0 - ? local.iam_additive - : {} - ) +resource "google_dataplex_datascan_iam_binding" "bindings" { + for_each = var.iam_bindings project = google_dataplex_datascan.datascan.project location = google_dataplex_datascan.datascan.location data_scan_id = google_dataplex_datascan.datascan.data_scan_id - role = each.value.role - member = each.value.member + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } -resource "google_dataplex_datascan_iam_member" "members" { - for_each = var.iam_members +resource "google_dataplex_datascan_iam_member" "bindings" { + for_each = var.iam_bindings_additive project = google_dataplex_datascan.datascan.project location = google_dataplex_datascan.datascan.location data_scan_id = google_dataplex_datascan.datascan.data_scan_id @@ -85,22 +72,3 @@ resource "google_dataplex_datascan_iam_member" "members" { } } } - -resource "google_dataplex_datascan_iam_policy" "authoritative_for_resource" { - count = var.iam_policy != null ? 1 : 0 - project = google_dataplex_datascan.datascan.project - location = google_dataplex_datascan.datascan.location - data_scan_id = google_dataplex_datascan.datascan.data_scan_id - policy_data = data.google_iam_policy.authoritative.0.policy_data -} - -data "google_iam_policy" "authoritative" { - count = var.iam_policy != null ? 1 : 0 - dynamic "binding" { - for_each = try(var.iam_policy, {}) - content { - role = binding.key - members = binding.value - } - } -} diff --git a/modules/dataplex-datascan/variables.tf b/modules/dataplex-datascan/variables.tf index f40ca4ec..4e6b2bb1 100644 --- a/modules/dataplex-datascan/variables.tf +++ b/modules/dataplex-datascan/variables.tf @@ -111,21 +111,22 @@ variable "iam" { nullable = false } -variable "iam_additive" { - description = "IAM additive bindings in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} - nullable = false +variable "iam_bindings" { + description = "Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_additive_members" { - description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values." - type = map(list(string)) - default = {} -} - -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string @@ -139,12 +140,6 @@ variable "iam_members" { default = {} } -variable "iam_policy" { - description = "IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution." - type = map(list(string)) - default = null -} - variable "incremental_field" { description = "The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table." type = string diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index b9dfe3d0..aa532671 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -8,7 +8,7 @@ This module Manages a Google Cloud [Dataproc](https://cloud.google.com/dataproc) - [Simple](#simple) - [Cluster configuration](#cluster-configuration) - [Cluster with CMEK encryption](#cluster-with-cmek-encryption) -- [IAM Examples](#iam-examples) +- [IAM](#iam) - [Authoritative IAM](#authoritative-iam) - [Additive IAM](#additive-iam) - [Variables](#variables) @@ -88,37 +88,20 @@ module "processing-dp-cluster" { # tftest modules=1 resources=1 ``` -## IAM Examples +## IAM -IAM is managed via several variables that implement different levels of control: +IAM is managed via several variables that implement different features and levels of control: -IAM is managed via several variables that implement different levels of control: +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions -- `group_iam` and `iam` configure authoritative bindings that manage individual roles exclusively, mapping to the [`google_project_iam_binding`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_binding) resource -- `iam_additive` and `iam_members` configures additive bindings that only manage individual role/member pairs, mapping to the [`google_project_iam_member`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_member) resource +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. + +Refer to the [project module](../project/README.md#iam) for examples of the IAM interface. ### Authoritative IAM -The iam variable is based on role keys and is typically used for service accounts, or where member values can be dynamic and would create potential problems in the underlying for_each cycle. - -```hcl -module "processing-dp-cluster" { - source = "./fabric/modules/dataproc" - project_id = "my-project" - name = "my-cluster" - region = "europe-west1" - prefix = "prefix" - iam = { - "roles/dataproc.viewer" = [ - "serviceAccount:service-account@PROJECT_ID.iam.gserviceaccount.com" - ] - } -} -# tftest modules=1 resources=2 -``` - -The group_iam variable uses group email addresses as keys and is a convenient way to assign roles to humans following Google's best practices. The end result is readable code that also serves as documentation. - ```hcl module "processing-dp-cluster" { source = "./fabric/modules/dataproc" @@ -131,14 +114,17 @@ module "processing-dp-cluster" { "roles/dataproc.viewer" ] } + iam = { + "roles/dataproc.viewer" = [ + "serviceAccount:service-account@PROJECT_ID.iam.gserviceaccount.com" + ] + } } # tftest modules=1 resources=2 ``` ### Additive IAM -Additive IAM is typically used where bindings for specific roles are controlled by different modules or in different Terraform stages. One example is when the cluster is created by one team but a different team manages access. - ```hcl module "processing-dp-cluster" { source = "./fabric/modules/dataproc" @@ -146,36 +132,31 @@ module "processing-dp-cluster" { name = "my-cluster" region = "europe-west1" prefix = "prefix" - iam_additive = { - "roles/dataproc.viewer" = [ - "serviceAccount:service-account@PROJECT_ID.iam.gserviceaccount.com" - ] - } - iam_members = { + iam_bindings_additive = { am1-viewer = { member = "user:am1@example.com" role = "roles/dataproc.viewer" } } } -# tftest modules=1 resources=3 +# tftest modules=1 resources=2 ``` ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L227) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L242) | Project ID. | string | ✓ | | -| [region](variables.tf#L247) | Dataproc region. | string | ✓ | | +| [name](variables.tf#L234) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L249) | Project ID. | string | ✓ | | +| [region](variables.tf#L254) | Dataproc region. | string | ✓ | | | [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | | [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L199) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_members](variables.tf#L206) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [labels](variables.tf#L221) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | -| [prefix](variables.tf#L232) | Optional prefix used to generate project id and name. | string | | null | -| [service_account](variables.tf#L252) | Service account to set on the Dataproc cluster. | string | | null | +| [iam_bindings](variables.tf#L199) | Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L213) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [labels](variables.tf#L228) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [prefix](variables.tf#L239) | Optional prefix used to generate project id and name. | string | | null | +| [service_account](variables.tf#L259) | Service account to set on the Dataproc cluster. | string | | null | ## Outputs diff --git a/modules/dataproc/iam.tf b/modules/dataproc/iam.tf index 04756192..fba2eca9 100644 --- a/modules/dataproc/iam.tf +++ b/modules/dataproc/iam.tf @@ -23,11 +23,6 @@ locals { for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null ] } - _iam_additive_pairs = flatten([ - for role, members in var.iam_additive : [ - for member in members : { role = role, member = member } - ] - ]) iam = { for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : role => concat( @@ -35,13 +30,6 @@ locals { try(local._group_iam[role], []) ) } - iam_additive = { - for pair in local._iam_additive_pairs : - "${pair.role}-${pair.member}" => { - role = pair.role - member = pair.member - } - } } resource "google_dataproc_cluster_iam_binding" "authoritative" { @@ -53,23 +41,28 @@ resource "google_dataproc_cluster_iam_binding" "authoritative" { members = each.value } -resource "google_dataproc_cluster_iam_member" "additive" { - for_each = ( - length(var.iam_additive) > 0 - ? local.iam_additive - : {} - ) - project = var.project_id - cluster = google_dataproc_cluster.cluster.name - region = var.region - role = each.value.role - member = each.value.member -} - -resource "google_dataproc_cluster_iam_member" "members" { - for_each = var.iam_members +resource "google_dataproc_cluster_iam_binding" "bindings" { + for_each = var.iam_bindings project = var.project_id cluster = google_dataproc_cluster.cluster.name + region = var.region + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_dataproc_cluster_iam_member" "bindings" { + for_each = var.iam_bindings_additive + project = var.project_id + cluster = google_dataproc_cluster.cluster.name + region = var.region role = each.value.role member = each.value.member dynamic "condition" { diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index 37aad39f..49f4fa90 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -196,15 +196,22 @@ variable "iam" { nullable = false } -variable "iam_additive" { - description = "IAM additive bindings in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} - nullable = false +variable "iam_bindings" { + description = "Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string diff --git a/modules/folder/README.md b/modules/folder/README.md index 03c2a755..9b998a11 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -33,35 +33,27 @@ module "folder" { iam = { "roles/owner" = ["user:one@example.org"] } - iam_additive = { - "roles/compute.admin" = ["user:a1@example.org", "user:a2@example.org"] - "roles/compute.viewer" = ["user:a2@example.org"] - } - iam_additive_members = { - "user:am1@example.org" = ["roles/storage.admin"] - "user:am2@example.org" = ["roles/storage.objectViewer"] - } - iam_members = { + iam_bindings_additive = { am1-storage-admin = { member = "user:am1@example.org" role = "roles/storage.admin" } } } -# tftest modules=1 resources=10 inventory=iam.yaml +# tftest modules=1 resources=5 inventory=iam.yaml ``` ## IAM -There are four three exclusive ways at the role level of managing IAM in this module +IAM is managed via several variables that implement different features and levels of control: -- non-authoritative via the `iam_additive`, `iam_additive_members` and `iam_members` variables, where bindings created outside this module will coexist with those managed here -- authoritative via the `group_iam` and `iam` variables, where bindings created outside this module (eg in the console) will be removed at each `terraform apply` cycle if the same role is also managed here -- authoritative policy via the `iam_policy` variable, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions -The authoritative and additive approaches can be used together, provided different roles are managed by each. The IAM policy is incompatible with the other approaches, and must be used with extreme care. +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. -Some care must be taken with the `groups_iam` variable (and in some situations with the additive variables) to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. +Refer to the [project module](../project/README.md#iam) for examples of the IAM interface. ## Organization policies @@ -241,32 +233,6 @@ module "folder" { # tftest modules=1 resources=3 inventory=logging-data-access.yaml ``` -While this sets an authoritative policies that has exclusive control of both IAM bindings for all roles and data access log configuration, and should be used with extreme care: - -```hcl -module "folder" { - source = "./fabric/modules/folder" - parent = "folders/657104291943" - name = "my-folder" - iam_policy = { - "roles/owner" = ["group:org-admins@example.com"] - "roles/resourcemanager.folderAdmin" = ["group:org-admins@example.com"] - "roles/resourcemanager.organizationAdmin" = ["group:org-admins@example.com"] - "roles/resourcemanager.projectCreator" = ["group:org-admins@example.com"] - } - logging_data_access = { - allServices = { - ADMIN_READ = ["group:organization-admins@example.org"] - } - "storage.googleapis.com" = { - DATA_READ = [] - DATA_WRITE = [] - } - } -} -# tftest modules=1 resources=2 inventory=iam-policy.yaml -``` - ## Tags Refer to the [Creating and managing tags](https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing) documentation for details on usage. @@ -305,7 +271,7 @@ module "folder" { | name | description | resources | |---|---|---| -| [iam.tf](./iam.tf) | IAM bindings, roles and audit logging resources. | google_folder_iam_binding · google_folder_iam_member · google_folder_iam_policy | +| [iam.tf](./iam.tf) | IAM bindings, roles and audit logging resources. | google_folder_iam_binding · google_folder_iam_member | | [logging.tf](./logging.tf) | Log sinks and supporting resources. | google_bigquery_dataset_iam_member · google_folder_iam_audit_config · google_logging_folder_exclusion · google_logging_folder_sink · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | | [main.tf](./main.tf) | Module-level locals and resources. | google_compute_firewall_policy_association · google_essential_contacts_contact · google_folder | | [organization-policies.tf](./organization-policies.tf) | Folder-level organization policies. | google_org_policy_policy | @@ -323,19 +289,17 @@ module "folder" { | [folder_create](variables.tf#L31) | Create folder. When set to false, uses id to reference an existing folder. | bool | | true | | [group_iam](variables.tf#L37) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L44) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L51) | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive_members](variables.tf#L58) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L65) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_policy](variables.tf#L80) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [id](variables.tf#L86) | Folder ID in case you use folder_create=false. | string | | null | -| [logging_data_access](variables.tf#L92) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L107) | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L114) | Logging sinks to create for the organization. | map(object({…})) | | {} | -| [name](variables.tf#L144) | Folder name. | string | | null | -| [org_policies](variables.tf#L150) | Organization policies applied to this folder keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L177) | Path containing org policies in YAML format. | string | | null | -| [parent](variables.tf#L183) | Parent in folders/folder_id or organizations/org_id format. | string | | null | -| [tag_bindings](variables.tf#L193) | Tag bindings for this folder, in key => tag value id format. | map(string) | | null | +| [iam_bindings](variables.tf#L51) | Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L65) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [id](variables.tf#L80) | Folder ID in case you use folder_create=false. | string | | null | +| [logging_data_access](variables.tf#L86) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L101) | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L108) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [name](variables.tf#L138) | Folder name. | string | | null | +| [org_policies](variables.tf#L144) | Organization policies applied to this folder keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L171) | Path containing org policies in YAML format. | string | | null | +| [parent](variables.tf#L177) | Parent in folders/folder_id or organizations/org_id format. | string | | null | +| [tag_bindings](variables.tf#L187) | Tag bindings for this folder, in key => tag value id format. | map(string) | | null | ## Outputs diff --git a/modules/folder/iam.tf b/modules/folder/iam.tf index cc457e28..976e312c 100644 --- a/modules/folder/iam.tf +++ b/modules/folder/iam.tf @@ -23,16 +23,6 @@ locals { for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null ] } - _iam_additive_pairs = flatten([ - for role, members in var.iam_additive : [ - for member in members : { role = role, member = member } - ] - ]) - _iam_additive_member_pairs = flatten([ - for member, roles in var.iam_additive_members : [ - for role in roles : { role = role, member = member } - ] - ]) iam = { for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : role => concat( @@ -40,10 +30,6 @@ locals { try(local._group_iam[role], []) ) } - iam_additive = { - for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) : - "${pair.role}-${pair.member}" => pair - } } resource "google_folder_iam_binding" "authoritative" { @@ -53,19 +39,23 @@ resource "google_folder_iam_binding" "authoritative" { members = each.value } -resource "google_folder_iam_member" "additive" { - for_each = ( - length(var.iam_additive) + length(var.iam_additive_members) > 0 - ? local.iam_additive - : {} - ) - folder = local.folder.name - role = each.value.role - member = each.value.member +resource "google_folder_iam_binding" "bindings" { + for_each = var.iam_bindings + folder = local.folder.name + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } -resource "google_folder_iam_member" "members" { - for_each = var.iam_members +resource "google_folder_iam_member" "bindings" { + for_each = var.iam_bindings_additive folder = local.folder.name role = each.value.role member = each.value.member @@ -78,34 +68,3 @@ resource "google_folder_iam_member" "members" { } } } - -resource "google_folder_iam_policy" "authoritative" { - count = var.iam_policy != null ? 1 : 0 - folder = local.folder.name - policy_data = data.google_iam_policy.authoritative.0.policy_data -} - -data "google_iam_policy" "authoritative" { - count = var.iam_policy != null ? 1 : 0 - dynamic "binding" { - for_each = try(var.iam_policy, {}) - content { - role = binding.key - members = binding.value - } - } - dynamic "audit_config" { - for_each = var.logging_data_access - content { - service = audit_config.key - dynamic "audit_log_configs" { - for_each = audit_config.value - iterator = config - content { - log_type = config.key - exempted_members = config.value - } - } - } - } -} diff --git a/modules/folder/logging.tf b/modules/folder/logging.tf index f7947f7a..8000a021 100644 --- a/modules/folder/logging.tf +++ b/modules/folder/logging.tf @@ -28,11 +28,9 @@ locals { } resource "google_folder_iam_audit_config" "default" { - for_each = ( - var.iam_policy == null ? var.logging_data_access : {} - ) - folder = local.folder.name - service = each.key + for_each = var.logging_data_access + folder = local.folder.name + service = each.key dynamic "audit_log_config" { for_each = each.value iterator = config diff --git a/modules/folder/outputs.tf b/modules/folder/outputs.tf index 2a791d74..3090bbeb 100644 --- a/modules/folder/outputs.tf +++ b/modules/folder/outputs.tf @@ -24,7 +24,7 @@ output "id" { value = local.folder.name depends_on = [ google_folder_iam_binding.authoritative, - google_folder_iam_policy.authoritative, + google_folder_iam_binding.bindings, google_org_policy_policy.default, ] } @@ -37,6 +37,7 @@ output "name" { output "sink_writer_identities" { description = "Writer identities created for each sink." value = { - for name, sink in google_logging_folder_sink.sink : name => sink.writer_identity + for name, sink in google_logging_folder_sink.sink : + name => sink.writer_identity } } diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf index df057995..0d914408 100644 --- a/modules/folder/variables.tf +++ b/modules/folder/variables.tf @@ -48,22 +48,22 @@ variable "iam" { nullable = false } -variable "iam_additive" { - description = "Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} - nullable = false +variable "iam_bindings" { + description = "Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_additive_members" { - description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values." - type = map(list(string)) - default = {} - nullable = false -} - -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string @@ -77,12 +77,6 @@ variable "iam_members" { default = {} } -variable "iam_policy" { - description = "IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution." - type = map(list(string)) - default = null -} - variable "id" { description = "Folder ID in case you use folder_create=false." type = string diff --git a/modules/iam-service-account/README.md b/modules/iam-service-account/README.md index 6c03b824..9fd6cba0 100644 --- a/modules/iam-service-account/README.md +++ b/modules/iam-service-account/README.md @@ -1,8 +1,12 @@ # Google Service Account Module -This module allows simplified creation and management of one a service account and its IAM bindings. A key can optionally be generated and will be stored in Terraform state. To use it create a sensitive output in your root modules referencing the `key` output, then extract the private key from the JSON formatted outputs. Alternatively, the `key` can be generated with `openssl` library and only public part uploaded to the Service Account, for more refer to the [Onprem SA Key Management](../../blueprints/cloud-operations/onprem-sa-key-management/) example. +This module allows simplified creation and management of one a service account and its IAM bindings. -Note that this module does not fully comply with our design principles, as outputs have no dependencies on IAM bindings to prevent resource cycles. +A key can optionally be generated and will be stored in Terraform state. To use it create a sensitive output in your root modules referencing the `key` output, then extract the private key from the JSON formatted outputs. + +Alternatively, the `key` can be generated with `openssl` library and only the public part uploaded to the Service Account, for more refer to the [Onprem SA Key Management](../../blueprints/cloud-operations/onprem-sa-key-management/) example. + +Note that outputs have no dependencies on IAM bindings to prevent resource cycles. ## Example @@ -41,23 +45,23 @@ module "myproject-default-service-accounts" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L106) | Name of the service account to create. | string | ✓ | | -| [project_id](variables.tf#L121) | Project id where service account will be created. | string | ✓ | | +| [name](variables.tf#L113) | Name of the service account to create. | string | ✓ | | +| [project_id](variables.tf#L128) | Project id where service account will be created. | string | ✓ | | | [description](variables.tf#L17) | Optional description. | string | | null | | [display_name](variables.tf#L23) | Display name of the service account to create. | string | | "Terraform-managed." | | [generate_key](variables.tf#L29) | Generate a key for service account. | bool | | false | | [iam](variables.tf#L35) | IAM bindings on the service account in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L42) | IAM additive bindings on the service account in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_billing_roles](variables.tf#L49) | Billing account roles granted to this service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | -| [iam_folder_roles](variables.tf#L56) | Folder roles granted to this service account, by folder id. Non-authoritative. | map(list(string)) | | {} | -| [iam_members](variables.tf#L63) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_organization_roles](variables.tf#L78) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | -| [iam_project_roles](variables.tf#L85) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | -| [iam_sa_roles](variables.tf#L92) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | -| [iam_storage_roles](variables.tf#L99) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | -| [prefix](variables.tf#L111) | Prefix applied to service account names. | string | | null | -| [public_keys_directory](variables.tf#L126) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | -| [service_account_create](variables.tf#L132) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | +| [iam_billing_roles](variables.tf#L42) | Billing account roles granted to this service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L49) | Authoritative IAM bindings on the service account in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L63) | Individual additive IAM bindings on the service account. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_folder_roles](variables.tf#L78) | Folder roles granted to this service account, by folder id. Non-authoritative. | map(list(string)) | | {} | +| [iam_organization_roles](variables.tf#L85) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | +| [iam_project_roles](variables.tf#L92) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | +| [iam_sa_roles](variables.tf#L99) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | +| [iam_storage_roles](variables.tf#L106) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | +| [prefix](variables.tf#L118) | Prefix applied to service account names. | string | | null | +| [public_keys_directory](variables.tf#L133) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | +| [service_account_create](variables.tf#L139) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | ## Outputs diff --git a/modules/iam-service-account/iam.tf b/modules/iam-service-account/iam.tf index a92d5e2a..a9423fb0 100644 --- a/modules/iam-service-account/iam.tf +++ b/modules/iam-service-account/iam.tf @@ -17,15 +17,6 @@ # tfdoc:file:description IAM bindings. locals { - _iam_additive_pairs = flatten([ - for role, members in var.iam_additive : [ - for member in members : { role = role, member = member } - ] - ]) - iam_additive = { - for pair in local._iam_additive_pairs : - "${pair.role}-${pair.member}" => pair - } iam_billing_pairs = flatten([ for entity, roles in var.iam_billing_roles : [ for role in roles : [ @@ -70,20 +61,43 @@ locals { ]) } -resource "google_service_account_iam_member" "roles" { - for_each = local.iam_additive - service_account_id = local.service_account.name - role = each.value.role - member = each.value.member -} - -resource "google_service_account_iam_binding" "roles" { +resource "google_service_account_iam_binding" "authoritative" { for_each = var.iam service_account_id = local.service_account.name role = each.key members = each.value } +resource "google_service_account_iam_binding" "bindings" { + for_each = var.iam_bindings + service_account_id = local.service_account.name + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_service_account_iam_member" "bindings" { + for_each = var.iam_bindings_additive + service_account_id = local.service_account.name + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + resource "google_billing_account_iam_member" "billing-roles" { for_each = { for pair in local.iam_billing_pairs : @@ -134,21 +148,6 @@ resource "google_service_account_iam_member" "additive" { member = local.resource_iam_email } -resource "google_service_account_iam_member" "members" { - for_each = var.iam_members - service_account_id = each.value.entity - role = each.value.role - member = each.value.member - dynamic "condition" { - for_each = each.value.condition == null ? [] : [""] - content { - expression = each.value.condition.expression - title = each.value.condition.title - description = each.value.condition.description - } - } -} - resource "google_storage_bucket_iam_member" "bucket-roles" { for_each = { for pair in local.iam_storage_pairs : diff --git a/modules/iam-service-account/variables.tf b/modules/iam-service-account/variables.tf index 87c350b6..c9ca7069 100644 --- a/modules/iam-service-account/variables.tf +++ b/modules/iam-service-account/variables.tf @@ -39,13 +39,6 @@ variable "iam" { nullable = false } -variable "iam_additive" { - description = "IAM additive bindings on the service account in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} - nullable = false -} - variable "iam_billing_roles" { description = "Billing account roles granted to this service account, by billing account id. Non-authoritative." type = map(list(string)) @@ -53,15 +46,22 @@ variable "iam_billing_roles" { nullable = false } -variable "iam_folder_roles" { - description = "Folder roles granted to this service account, by folder id. Non-authoritative." - type = map(list(string)) - default = {} - nullable = false +variable "iam_bindings" { + description = "Authoritative IAM bindings on the service account in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings on the service account. Keys are arbitrary." type = map(object({ member = string role = string @@ -75,6 +75,13 @@ variable "iam_members" { default = {} } +variable "iam_folder_roles" { + description = "Folder roles granted to this service account, by folder id. Non-authoritative." + type = map(list(string)) + default = {} + nullable = false +} + variable "iam_organization_roles" { description = "Organization roles granted to this service account, by organization id. Non-authoritative." type = map(list(string)) diff --git a/modules/kms/README.md b/modules/kms/README.md index 5bd82d5f..56acff46 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -42,24 +42,12 @@ module "kms" { module "kms" { source = "./fabric/modules/kms" project_id = "my-project" - iam_additive = { - "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ - "user:user1@example.com", "user:user2@example.com" - ] - } key_iam = { key-a = { "roles/cloudkms.admin" = ["user:user3@example.com"] } } - key_iam_additive = { - key-b = { - "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ - "user:user4@example.com", "user:user5@example.com" - ] - } - } - key_iam_members = { + key_iam_bindings_additive = { key-b-am1 = { key = "key-b" member = "user:am1@example.com" @@ -73,7 +61,7 @@ module "kms" { key-c = { rotation_period = null, labels = { env = "test" } } } } -# tftest modules=1 resources=10 +# tftest modules=1 resources=6 ``` ### Crypto key purpose @@ -101,28 +89,28 @@ module "kms" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [keyring](variables.tf#L101) | Keyring attributes. | object({…}) | ✓ | | -| [project_id](variables.tf#L124) | Project id where the keyring will be created. | string | ✓ | | +| [keyring](variables.tf#L117) | Keyring attributes. | object({…}) | ✓ | | +| [project_id](variables.tf#L140) | Project id where the keyring will be created. | string | ✓ | | | [iam](variables.tf#L17) | Keyring IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L23) | Keyring IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_members](variables.tf#L29) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [key_iam](variables.tf#L44) | Key IAM bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [key_iam_additive](variables.tf#L50) | Key IAM additive bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [key_iam_members](variables.tf#L56) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [key_purpose](variables.tf#L72) | Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | map(object({…})) | | {} | -| [key_purpose_defaults](variables.tf#L84) | Defaults used for key purpose when not defined at the key level. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | object({…}) | | {…} | -| [keyring_create](variables.tf#L109) | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | -| [keys](variables.tf#L115) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | -| [tag_bindings](variables.tf#L129) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | null | +| [iam_bindings](variables.tf#L23) | Keyring authoritative IAM bindings in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L37) | Keyring individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [key_iam](variables.tf#L52) | Key IAM bindings in {KEY => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [key_iam_bindings](variables.tf#L58) | Key authoritative IAM bindings in {KEY => {ROLE => {members = [], condition = {}}}}. | map(object({…})) | | {} | +| [key_iam_bindings_additive](variables.tf#L72) | Key individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [key_purpose](variables.tf#L88) | Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | map(object({…})) | | {} | +| [key_purpose_defaults](variables.tf#L100) | Defaults used for key purpose when not defined at the key level. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | object({…}) | | {…} | +| [keyring_create](variables.tf#L125) | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | +| [keys](variables.tf#L131) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L145) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | null | ## Outputs | name | description | sensitive | |---|---|:---:| | [id](outputs.tf#L17) | Fully qualified keyring id. | | -| [key_ids](outputs.tf#L25) | Fully qualified key ids. | | -| [keyring](outputs.tf#L36) | Keyring resource. | | -| [keys](outputs.tf#L44) | Key resources. | | -| [location](outputs.tf#L52) | Keyring location. | | -| [name](outputs.tf#L60) | Keyring name. | | +| [key_ids](outputs.tf#L26) | Fully qualified key ids. | | +| [keyring](outputs.tf#L38) | Keyring resource. | | +| [keys](outputs.tf#L47) | Key resources. | | +| [location](outputs.tf#L56) | Keyring location. | | +| [name](outputs.tf#L65) | Keyring name. | | diff --git a/modules/kms/iam.tf b/modules/kms/iam.tf index 9289a4e8..9a78c2cf 100644 --- a/modules/kms/iam.tf +++ b/modules/kms/iam.tf @@ -15,26 +15,7 @@ */ locals { - iam_additive_members = flatten([ - for role, members in var.iam_additive : [ - for member in members : { - member = member - role = role - } - ] - ]) - key_iam_additive_members = flatten([ - for key, roles in var.key_iam_additive : [ - for role, members in roles : [ - for member in members : { - key = key - member = member - role = role - } - ] - ] - ]) - key_iam_members = flatten([ + key_iam = flatten([ for key, roles in var.key_iam : [ for role, members in roles : { key = key @@ -43,27 +24,42 @@ locals { } ] ]) + key_iam_bindings = flatten([ + for key, roles in var.key_iam_bindings : [ + for role, data in roles : { + key = key + role = role + members = data.members + condition = data.condition + } + ] + ]) } -resource "google_kms_key_ring_iam_binding" "default" { +resource "google_kms_key_ring_iam_binding" "authoritative" { for_each = var.iam key_ring_id = local.keyring.id role = each.key members = each.value } -resource "google_kms_key_ring_iam_member" "default" { - for_each = { - for binding in local.iam_additive_members : - "${binding.role}${binding.member}" => binding - } +resource "google_kms_key_ring_iam_binding" "bindings" { + for_each = var.iam_bindings key_ring_id = local.keyring.id - role = each.value.role - member = each.value.member + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } -resource "google_kms_key_ring_iam_member" "members" { - for_each = var.iam_members +resource "google_kms_key_ring_iam_member" "bindings" { + for_each = var.iam_bindings_additive key_ring_id = local.keyring.id role = each.value.role member = each.value.member @@ -77,9 +73,9 @@ resource "google_kms_key_ring_iam_member" "members" { } } -resource "google_kms_crypto_key_iam_binding" "default" { +resource "google_kms_crypto_key_iam_binding" "authoritative" { for_each = { - for binding in local.key_iam_members : + for binding in local.key_iam : "${binding.key}.${binding.role}" => binding } role = each.value.role @@ -87,18 +83,26 @@ resource "google_kms_crypto_key_iam_binding" "default" { members = each.value.members } -resource "google_kms_crypto_key_iam_member" "default" { +resource "google_kms_crypto_key_iam_binding" "bindings" { for_each = { - for binding in local.key_iam_additive_members : - "${binding.key}.${binding.role}${binding.member}" => binding + for binding in local.key_iam_bindings : + "${binding.key}.${binding.role}" => binding } role = each.value.role crypto_key_id = google_kms_crypto_key.default[each.value.key].id - member = each.value.member + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } resource "google_kms_crypto_key_iam_member" "members" { - for_each = var.key_iam_members + for_each = var.key_iam_bindings_additive crypto_key_id = google_kms_crypto_key.default[each.value.key].id role = each.value.role member = each.value.member diff --git a/modules/kms/outputs.tf b/modules/kms/outputs.tf index b3e6d76a..191db82b 100644 --- a/modules/kms/outputs.tf +++ b/modules/kms/outputs.tf @@ -18,7 +18,8 @@ output "id" { description = "Fully qualified keyring id." value = local.keyring.id depends_on = [ - google_kms_key_ring_iam_binding.default + google_kms_key_ring_iam_binding.authoritative, + google_kms_key_ring_iam_binding.bindings ] } @@ -29,7 +30,8 @@ output "key_ids" { name => resource.id } depends_on = [ - google_kms_crypto_key_iam_binding.default + google_kms_crypto_key_iam_binding.authoritative, + google_kms_crypto_key_iam_binding.bindings ] } @@ -37,7 +39,8 @@ output "keyring" { description = "Keyring resource." value = local.keyring depends_on = [ - google_kms_key_ring_iam_binding.default + google_kms_key_ring_iam_binding.authoritative, + google_kms_key_ring_iam_binding.bindings ] } @@ -45,7 +48,8 @@ output "keys" { description = "Key resources." value = google_kms_crypto_key.default depends_on = [ - google_kms_crypto_key_iam_binding.default + google_kms_crypto_key_iam_binding.authoritative, + google_kms_crypto_key_iam_binding.bindings ] } @@ -53,7 +57,8 @@ output "location" { description = "Keyring location." value = local.keyring.location depends_on = [ - google_kms_key_ring_iam_binding.default + google_kms_key_ring_iam_binding.authoritative, + google_kms_key_ring_iam_binding.bindings ] } @@ -61,6 +66,7 @@ output "name" { description = "Keyring name." value = local.keyring.name depends_on = [ - google_kms_key_ring_iam_binding.default + google_kms_key_ring_iam_binding.authoritative, + google_kms_key_ring_iam_binding.bindings ] } diff --git a/modules/kms/variables.tf b/modules/kms/variables.tf index c34aaaac..44c98036 100644 --- a/modules/kms/variables.tf +++ b/modules/kms/variables.tf @@ -20,14 +20,22 @@ variable "iam" { default = {} } -variable "iam_additive" { - description = "Keyring IAM additive bindings in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} +variable "iam_bindings" { + description = "Keyring authoritative IAM bindings in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Keyring individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string @@ -47,14 +55,22 @@ variable "key_iam" { default = {} } -variable "key_iam_additive" { - description = "Key IAM additive bindings in {KEY => {ROLE => [MEMBERS]}} format." - type = map(map(list(string))) - default = {} +variable "key_iam_bindings" { + description = "Key authoritative IAM bindings in {KEY => {ROLE => {members = [], condition = {}}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "key_iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "key_iam_bindings_additive" { + description = "Key individual additive IAM bindings. Keys are arbitrary." type = map(object({ key = string member = string diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 0ccfeb86..3aaaa2a7 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -100,6 +100,8 @@ module "vpc" { ### Subnet IAM +Subnet IAM variables follow our general interface, with extra keys/members for the subnet to which each binding will be applied. + ```hcl module "vpc" { source = "./fabric/modules/net-vpc" @@ -124,14 +126,18 @@ module "vpc" { ] } } - subnet_iam_additive = { - "europe-west1/subnet-2" = { - "roles/compute.networkUser" = [ - "user:user2@example.com", "group:group2@example.com" - ] + subnet_iam_bindings = { + "europe-west1/subnet-1" = { + "roles/compute.networkUser" = { + members = ["group:group2@example.com"] + condition = { + expression = "resource.matchTag('123456789012/env', 'prod')" + title = "test_condition" + } + } } } - subnet_iam_members = { + subnet_iam_bindings_additive = { subnet-2-am1 = { member = "user:am1@example.com" role = "roles/compute.networkUser" @@ -139,7 +145,7 @@ module "vpc" { } } } -# tftest modules=1 resources=9 inventory=subnet-iam.yaml +# tftest modules=1 resources=8 inventory=subnet-iam.yaml ``` ### Peering @@ -342,7 +348,7 @@ module "vpc" { name = "my-network" data_folder = "config/subnets" } -# tftest modules=1 resources=11 files=subnet-simple,subnet-simple-2,subnet-detailed,subnet-proxy,subnet-psc inventory=factory.yaml +# tftest modules=1 resources=9 files=subnet-simple,subnet-simple-2,subnet-detailed,subnet-proxy,subnet-psc inventory=factory.yaml ``` ```yaml @@ -370,9 +376,6 @@ iam: # grant roles/compute.networkUser - group:lorem@example.com - serviceAccount:fbz@prj.iam.gserviceaccount.com - user:foobar@example.com -iam_additive: # grant roles/compute.networkUser - - user:foo@example.com - - serviceAccount:fbx@prj.iam.gserviceaccount.com secondary_ip_ranges: # map of secondary ip ranges secondary-range-a: 192.168.0.0/24 flow_logs: # enable, set to empty map to use defaults @@ -540,31 +543,30 @@ module "vpc" { | [shared_vpc_host](variables.tf#L155) | Enable shared VPC for this project. | bool | | false | | [shared_vpc_service_projects](variables.tf#L161) | Shared VPC service projects to register with this host. | list(string) | | [] | | [subnet_iam](variables.tf#L167) | Subnet IAM bindings in {REGION/NAME => {ROLE => [MEMBERS]} format. | map(map(list(string))) | | {} | -| [subnet_iam_additive](variables.tf#L173) | Subnet IAM additive bindings in {REGION/NAME => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [subnet_iam_members](variables.tf#L180) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [subnets](variables.tf#L196) | Subnet configuration. | list(object({…})) | | [] | -| [subnets_proxy_only](variables.tf#L222) | List of proxy-only subnets for Regional HTTPS or Internal HTTPS load balancers. Note: Only one proxy-only subnet for each VPC network in each region can be active. | list(object({…})) | | [] | -| [subnets_psc](variables.tf#L234) | List of subnets for Private Service Connect service producers. | list(object({…})) | | [] | -| [vpc_create](variables.tf#L245) | Create VPC. When set to false, uses a data source to reference existing VPC. | bool | | true | +| [subnet_iam_bindings](variables.tf#L173) | Authoritative IAM bindings in {REGION/NAME => {ROLE => {members = [], condition = {}}}}. | map(map(object({…}))) | | {} | +| [subnet_iam_bindings_additive](variables.tf#L187) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [subnets](variables.tf#L203) | Subnet configuration. | list(object({…})) | | [] | +| [subnets_proxy_only](variables.tf#L229) | List of proxy-only subnets for Regional HTTPS or Internal HTTPS load balancers. Note: Only one proxy-only subnet for each VPC network in each region can be active. | list(object({…})) | | [] | +| [subnets_psc](variables.tf#L241) | List of subnets for Private Service Connect service producers. | list(object({…})) | | [] | +| [vpc_create](variables.tf#L252) | Create VPC. When set to false, uses a data source to reference existing VPC. | bool | | true | ## Outputs | name | description | sensitive | |---|---|:---:| -| [bindings](outputs.tf#L17) | Subnet IAM bindings. | | -| [id](outputs.tf#L22) | Fully qualified network id. | | -| [internal_ipv6_range](outputs.tf#L34) | ULA range. | | -| [name](outputs.tf#L39) | Network name. | | -| [network](outputs.tf#L51) | Network resource. | | -| [project_id](outputs.tf#L63) | Project ID containing the network. Use this when you need to create resources *after* the VPC is fully set up (e.g. subnets created, shared VPC service projects attached, Private Service Networking configured). | | -| [self_link](outputs.tf#L76) | Network self link. | | -| [subnet_ids](outputs.tf#L88) | Map of subnet IDs keyed by name. | | -| [subnet_ips](outputs.tf#L93) | Map of subnet address ranges keyed by name. | | -| [subnet_ipv6_external_prefixes](outputs.tf#L100) | Map of subnet external IPv6 prefixes keyed by name. | | -| [subnet_regions](outputs.tf#L108) | Map of subnet regions keyed by name. | | -| [subnet_secondary_ranges](outputs.tf#L115) | Map of subnet secondary ranges keyed by name. | | -| [subnet_self_links](outputs.tf#L126) | Map of subnet self links keyed by name. | | -| [subnets](outputs.tf#L131) | Subnet resources. | | -| [subnets_proxy_only](outputs.tf#L136) | L7 ILB or L7 Regional LB subnet resources. | | -| [subnets_psc](outputs.tf#L141) | Private Service Connect subnet resources. | | +| [id](outputs.tf#L17) | Fully qualified network id. | | +| [internal_ipv6_range](outputs.tf#L29) | ULA range. | | +| [name](outputs.tf#L34) | Network name. | | +| [network](outputs.tf#L46) | Network resource. | | +| [project_id](outputs.tf#L58) | Project ID containing the network. Use this when you need to create resources *after* the VPC is fully set up (e.g. subnets created, shared VPC service projects attached, Private Service Networking configured). | | +| [self_link](outputs.tf#L71) | Network self link. | | +| [subnet_ids](outputs.tf#L83) | Map of subnet IDs keyed by name. | | +| [subnet_ips](outputs.tf#L88) | Map of subnet address ranges keyed by name. | | +| [subnet_ipv6_external_prefixes](outputs.tf#L95) | Map of subnet external IPv6 prefixes keyed by name. | | +| [subnet_regions](outputs.tf#L103) | Map of subnet regions keyed by name. | | +| [subnet_secondary_ranges](outputs.tf#L110) | Map of subnet secondary ranges keyed by name. | | +| [subnet_self_links](outputs.tf#L121) | Map of subnet self links keyed by name. | | +| [subnets](outputs.tf#L126) | Subnet resources. | | +| [subnets_proxy_only](outputs.tf#L131) | L7 ILB or L7 Regional LB subnet resources. | | +| [subnets_psc](outputs.tf#L136) | Private Service Connect subnet resources. | | diff --git a/modules/net-vpc/outputs.tf b/modules/net-vpc/outputs.tf index 69c659a2..fbf07dba 100644 --- a/modules/net-vpc/outputs.tf +++ b/modules/net-vpc/outputs.tf @@ -14,11 +14,6 @@ * limitations under the License. */ -output "bindings" { - description = "Subnet IAM bindings." - value = { for k, v in google_compute_subnetwork_iam_binding.binding : k => v } -} - output "id" { description = "Fully qualified network id." value = local.network.id diff --git a/modules/net-vpc/subnets.tf b/modules/net-vpc/subnets.tf index b6eab182..0e656fd8 100644 --- a/modules/net-vpc/subnets.tf +++ b/modules/net-vpc/subnets.tf @@ -32,20 +32,11 @@ locals { ipv6 = try(v.ipv6, null) secondary_ip_ranges = try(v.secondary_ip_ranges, null) iam = try(v.iam, []) - iam_additive = try(v.iam_additive, []) + iam_members = try(v.iam_members, []) purpose = try(v.purpose, null) active = try(v.active, null) } } - _factory_subnets_iam_additive = flatten([ - for k, v in local._factory_subnets : [ - for member in lookup(v, "iam_additive", []) : { - member = member - subnet = k - role = "roles/compute.networkUser" - } - ] if v.purpose == null - ]) _factory_subnets_iam = [ for k, v in local._factory_subnets : { subnet = k @@ -53,18 +44,7 @@ locals { members = v.iam } if v.purpose == null && v.iam != null ] - _subnet_iam_additive_members = flatten([ - for subnet, roles in var.subnet_iam_additive : [ - for role, members in roles : [ - for member in members : { - member = member - role = role - subnet = subnet - } - ] - ] - ]) - _subnet_iam_members = flatten([ + _subnet_iam = flatten([ for subnet, roles in(var.subnet_iam == null ? {} : var.subnet_iam) : [ for role, members in roles : { members = members @@ -73,14 +53,20 @@ locals { } ] ]) - subnet_iam_additive_members = concat( - local._factory_subnets_iam_additive, - local._subnet_iam_additive_members - ) - subnet_iam_members = concat( + subnet_iam = concat( [for k in local._factory_subnets_iam : k if length(k.members) > 0], - local._subnet_iam_members + local._subnet_iam ) + subnet_iam_bindings = flatten([ + for subnet, roles in(var.subnet_iam_bindings == null ? {} : var.subnet_iam_bindings) : [ + for role, data in roles : { + role = role + subnet = subnet + members = data.members + condition = data.condition + } + ] + ]) subnets = merge( { for s in var.subnets : "${s.region}/${s.name}" => s }, { for k, v in local._factory_subnets : k => v if v.purpose == null } @@ -166,9 +152,9 @@ resource "google_compute_subnetwork" "psc" { purpose = "PRIVATE_SERVICE_CONNECT" } -resource "google_compute_subnetwork_iam_binding" "binding" { +resource "google_compute_subnetwork_iam_binding" "authoritative" { for_each = { - for binding in local.subnet_iam_members : + for binding in local.subnet_iam : "${binding.subnet}.${binding.role}" => binding } project = var.project_id @@ -178,20 +164,30 @@ resource "google_compute_subnetwork_iam_binding" "binding" { members = each.value.members } -resource "google_compute_subnetwork_iam_member" "binding" { +resource "google_compute_subnetwork_iam_binding" "bindings" { for_each = { - for binding in local.subnet_iam_additive_members : - "${binding.subnet}.${binding.role}.${binding.member}" => binding + for binding in local.subnet_iam_bindings : + "${binding.subnet}.${binding.role}.${try(binding.condition.title, "")}" => binding } project = var.project_id subnetwork = google_compute_subnetwork.subnetwork[each.value.subnet].name region = google_compute_subnetwork.subnetwork[each.value.subnet].region role = each.value.role - member = each.value.member + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } -resource "google_compute_subnetwork_iam_member" "members" { - for_each = var.subnet_iam_members +# TODO: merge factory subnet IAM members + +resource "google_compute_subnetwork_iam_member" "bindings" { + for_each = var.subnet_iam_bindings_additive project = var.project_id subnetwork = google_compute_subnetwork.subnetwork[each.value.subnet].name region = google_compute_subnetwork.subnetwork[each.value.subnet].region diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index c49455fa..3837c9b0 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -170,15 +170,22 @@ variable "subnet_iam" { default = {} } -variable "subnet_iam_additive" { - description = "Subnet IAM additive bindings in {REGION/NAME => {ROLE => [MEMBERS]}} format." - type = map(map(list(string))) - default = {} - nullable = false +variable "subnet_iam_bindings" { + description = "Authoritative IAM bindings in {REGION/NAME => {ROLE => {members = [], condition = {}}}}." + type = map(map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + }))) + nullable = false + default = {} } -variable "subnet_iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "subnet_iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string diff --git a/modules/organization/README.md b/modules/organization/README.md index 4984d2a8..16507800 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -42,10 +42,7 @@ module "org" { iam = { "roles/resourcemanager.projectCreator" = ["group:cloud-admins@example.org"] } - iam_additive_members = { - "user:compute@example.org" = ["roles/compute.admin", "roles/container.viewer"] - } - iam_members = { + iam_bindings_additive = { am1-storage-admin = { member = "user:am1@example.org" role = "roles/storage.admin" @@ -121,20 +118,20 @@ module "org" { } } } -# tftest modules=1 resources=17 inventory=basic.yaml +# tftest modules=1 resources=15 inventory=basic.yaml ``` ## IAM -There are three mutually exclusive ways at the role level of managing IAM in this module +IAM is managed via several variables that implement different features and levels of control: -- non-authoritative via the `iam_additive`, `iam_additive_members` and `iam_members` variables, where bindings created outside this module will coexist with those managed here -- authoritative via the `group_iam` and `iam` variables, where bindings created outside this module (eg in the console) will be removed at each `terraform apply` cycle if the same role is also managed here -- authoritative policy via the `iam_policy` variable, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions -The authoritative and additive approaches can be used together, provided different roles are managed by each. The IAM policy is incompatible with the other approaches, and must be used with extreme care. +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. -Some care must also be taken with the `group_iam` variable (and in some situations with the additive variables) to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. +Refer to the [project module](../project/README.md#iam) for examples of the IAM interface. ## Organization Policies @@ -327,9 +324,7 @@ module "org" { ## Data Access Logs -Activation of data access logs can be controlled via the `logging_data_access` variable. If the `iam_bindings_authoritative` variable is used to set a resource-level IAM policy, the data access log configuration will also be authoritative as part of the policy. - -This example shows how to set a non-authoritative access log configuration: +Activation of data access logs can be controlled via the `logging_data_access` variable. ```hcl module "org" { @@ -349,31 +344,6 @@ module "org" { # tftest modules=1 resources=2 inventory=logging-data-access.yaml ``` -While this sets an authoritative policies that has exclusive control of both IAM bindings for all roles and data access log configuration, and should be used with extreme care: - -```hcl -module "org" { - source = "./fabric/modules/organization" - organization_id = var.organization_id - iam_policy = { - "roles/owner" = ["group:org-admins@example.com"] - "roles/resourcemanager.folderAdmin" = ["group:org-admins@example.com"] - "roles/resourcemanager.organizationAdmin" = ["group:org-admins@example.com"] - "roles/resourcemanager.projectCreator" = ["group:org-admins@example.com"] - } - logging_data_access = { - allServices = { - ADMIN_READ = ["group:organization-admins@example.org"] - } - "storage.googleapis.com" = { - DATA_READ = [] - DATA_WRITE = [] - } - } -} -# tftest modules=1 resources=1 inventory=iam-policy.yaml -``` - ## Custom Roles Custom roles can be defined via the `custom_roles` variable, and referenced via the `custom_role_id` output: @@ -461,7 +431,7 @@ module "org" { | name | description | resources | |---|---|---| -| [iam.tf](./iam.tf) | IAM bindings, roles and audit logging resources. | google_organization_iam_binding · google_organization_iam_custom_role · google_organization_iam_member · google_organization_iam_policy | +| [iam.tf](./iam.tf) | IAM bindings, roles and audit logging resources. | google_organization_iam_binding · google_organization_iam_custom_role · google_organization_iam_member | | [logging.tf](./logging.tf) | Log sinks and data access logs. | google_bigquery_dataset_iam_member · google_logging_organization_exclusion · google_logging_organization_sink · google_organization_iam_audit_config · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | | [main.tf](./main.tf) | Module-level locals and resources. | google_compute_firewall_policy_association · google_essential_contacts_contact | | [org-policy-custom-constraints.tf](./org-policy-custom-constraints.tf) | None | google_org_policy_custom_constraint | @@ -475,26 +445,24 @@ module "org" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [organization_id](variables.tf#L214) | Organization id in organizations/nnnnnn format. | string | ✓ | | +| [organization_id](variables.tf#L208) | Organization id in organizations/nnnnnn format. | string | ✓ | | | [contacts](variables.tf#L17) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | | [custom_roles](variables.tf#L24) | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | | [firewall_policy_associations](variables.tf#L31) | Hierarchical firewall policies to associate to this folder, in association name => policy id format. | map(string) | | {} | | [group_iam](variables.tf#L38) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L45) | IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L52) | Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive_members](variables.tf#L59) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L66) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_policy](variables.tf#L81) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [logging_data_access](variables.tf#L87) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L102) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L109) | Logging sinks to create for the organization. | map(object({…})) | | {} | -| [network_tags](variables.tf#L139) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | -| [org_policies](variables.tf#L161) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L188) | Path containing org policies in YAML format. | string | | null | -| [org_policy_custom_constraints](variables.tf#L194) | Organization policy custom constraints keyed by constraint name. | map(object({…})) | | {} | -| [org_policy_custom_constraints_data_path](variables.tf#L208) | Path containing org policy custom constraints in YAML format. | string | | null | -| [tag_bindings](variables.tf#L223) | Tag bindings for this organization, in key => tag value id format. | map(string) | | null | -| [tags](variables.tf#L229) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | +| [iam_bindings](variables.tf#L52) | Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L66) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [logging_data_access](variables.tf#L81) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L96) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L103) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [network_tags](variables.tf#L133) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | +| [org_policies](variables.tf#L155) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L182) | Path containing org policies in YAML format. | string | | null | +| [org_policy_custom_constraints](variables.tf#L188) | Organization policy custom constraints keyed by constraint name. | map(object({…})) | | {} | +| [org_policy_custom_constraints_data_path](variables.tf#L202) | Path containing org policy custom constraints in YAML format. | string | | null | +| [tag_bindings](variables.tf#L217) | Tag bindings for this organization, in key => tag value id format. | map(string) | | null | +| [tags](variables.tf#L223) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | ## Outputs diff --git a/modules/organization/iam.tf b/modules/organization/iam.tf index e5ba3771..2882d02a 100644 --- a/modules/organization/iam.tf +++ b/modules/organization/iam.tf @@ -23,16 +23,6 @@ locals { for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null ] } - _iam_additive_pairs = flatten([ - for role, members in var.iam_additive : [ - for member in members : { role = role, member = member } - ] - ]) - _iam_additive_member_pairs = flatten([ - for member, roles in var.iam_additive_members : [ - for role in roles : { role = role, member = member } - ] - ]) iam = { for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : role => concat( @@ -40,10 +30,6 @@ locals { try(local._group_iam[role], []) ) } - iam_additive = { - for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) : - "${pair.role}-${pair.member}" => pair - } } resource "google_organization_iam_custom_role" "roles" { @@ -62,19 +48,23 @@ resource "google_organization_iam_binding" "authoritative" { members = each.value } -resource "google_organization_iam_member" "additive" { - for_each = ( - length(var.iam_additive) + length(var.iam_additive_members) > 0 - ? local.iam_additive - : {} - ) - org_id = local.organization_id_numeric - role = each.value.role - member = each.value.member +resource "google_organization_iam_binding" "bindings" { + for_each = var.iam_bindings + org_id = local.organization_id_numeric + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } -resource "google_organization_iam_member" "members" { - for_each = var.iam_members +resource "google_organization_iam_member" "bindings" { + for_each = var.iam_bindings_additive org_id = local.organization_id_numeric role = each.value.role member = each.value.member @@ -87,34 +77,3 @@ resource "google_organization_iam_member" "members" { } } } - -resource "google_organization_iam_policy" "authoritative" { - count = var.iam_policy != null ? 1 : 0 - org_id = local.organization_id_numeric - policy_data = data.google_iam_policy.authoritative.0.policy_data -} - -data "google_iam_policy" "authoritative" { - count = var.iam_policy != null ? 1 : 0 - dynamic "binding" { - for_each = try(var.iam_policy, {}) - content { - role = binding.key - members = binding.value - } - } - dynamic "audit_config" { - for_each = var.logging_data_access - content { - service = audit_config.key - dynamic "audit_log_configs" { - for_each = audit_config.value - iterator = config - content { - log_type = config.key - exempted_members = config.value - } - } - } - } -} diff --git a/modules/organization/logging.tf b/modules/organization/logging.tf index 65e773a8..7719c0fb 100644 --- a/modules/organization/logging.tf +++ b/modules/organization/logging.tf @@ -27,11 +27,9 @@ locals { } resource "google_organization_iam_audit_config" "default" { - for_each = ( - var.iam_policy == null ? var.logging_data_access : {} - ) - org_id = local.organization_id_numeric - service = each.key + for_each = var.logging_data_access + org_id = local.organization_id_numeric + service = each.key dynamic "audit_log_config" { for_each = each.value iterator = config @@ -67,11 +65,10 @@ resource "google_logging_organization_sink" "sink" { filter = exclusion.value } } - depends_on = [ google_organization_iam_binding.authoritative, - google_organization_iam_member.additive, - google_organization_iam_policy.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings ] } diff --git a/modules/organization/organization-policies.tf b/modules/organization/organization-policies.tf index 6f10bb93..8d867f66 100644 --- a/modules/organization/organization-policies.tf +++ b/modules/organization/organization-policies.tf @@ -116,9 +116,9 @@ resource "google_org_policy_policy" "default" { } depends_on = [ google_organization_iam_binding.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings, google_organization_iam_custom_role.roles, - google_organization_iam_member.additive, - google_organization_iam_policy.authoritative, google_org_policy_custom_constraint.constraint, google_tags_tag_key.default, google_tags_tag_value.default, diff --git a/modules/organization/outputs.tf b/modules/organization/outputs.tf index 9c1ec187..12c133e8 100644 --- a/modules/organization/outputs.tf +++ b/modules/organization/outputs.tf @@ -44,9 +44,9 @@ output "id" { google_org_policy_custom_constraint.constraint, google_org_policy_policy.default, google_organization_iam_binding.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings, google_organization_iam_custom_role.roles, - google_organization_iam_member.additive, - google_organization_iam_policy.authoritative, google_tags_tag_key.default, google_tags_tag_key_iam_binding.default, google_tags_tag_value.default, @@ -80,9 +80,9 @@ output "organization_id" { google_org_policy_custom_constraint.constraint, google_org_policy_policy.default, google_organization_iam_binding.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings, google_organization_iam_custom_role.roles, - google_organization_iam_member.additive, - google_organization_iam_policy.authoritative, google_tags_tag_key.default, google_tags_tag_key_iam_binding.default, google_tags_tag_value.default, diff --git a/modules/organization/tags.tf b/modules/organization/tags.tf index b579479e..7fb1c068 100644 --- a/modules/organization/tags.tf +++ b/modules/organization/tags.tf @@ -81,8 +81,8 @@ resource "google_tags_tag_key" "default" { description = each.value.description depends_on = [ google_organization_iam_binding.authoritative, - google_organization_iam_member.additive, - google_organization_iam_policy.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings ] } diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf index 2426f8e9..cf59cae6 100644 --- a/modules/organization/variables.tf +++ b/modules/organization/variables.tf @@ -49,22 +49,22 @@ variable "iam" { nullable = false } -variable "iam_additive" { - description = "Non authoritative IAM bindings, in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} - nullable = false +variable "iam_bindings" { + description = "Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_additive_members" { - description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values." - type = map(list(string)) - default = {} - nullable = false -} - -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string @@ -78,12 +78,6 @@ variable "iam_members" { default = {} } -variable "iam_policy" { - description = "IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution." - type = map(list(string)) - default = null -} - variable "logging_data_access" { description = "Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services." type = map(map(list(string))) diff --git a/modules/project/README.md b/modules/project/README.md index 8ec483fb..7479eca8 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -10,11 +10,7 @@ This module implements the creation and management of one GCP project including - [IAM](#iam) - [Authoritative IAM](#authoritative-iam) - [Additive IAM](#additive-iam) - - [Additive IAM by Role](#additive-iam-by-role) - - [Additive IAM by Principal](#additive-iam-by-principal) - - [Additive IAM by Binding](#additive-iam-by-binding) - [Service Identities and Authoritative IAM](#service-identities-and-authoritative-iam) - - [Using Shortcodes for Service Identities in Additive Iam](#using-shortcodes-for-service-identities-in-additive-iam) - [Service Identities Requiring Manual Iam Grants](#service-identities-requiring-manual-iam-grants) - [Shared VPC](#shared-vpc) - [Organization Policies](#organization-policies) @@ -48,13 +44,13 @@ module "project" { ## IAM -IAM is managed via several variables that implement different levels of control: +IAM is managed via several variables that implement different features and levels of control: -- `group_iam` and `iam` configure authoritative bindings that manage individual roles exclusively, mapping to the [`google_project_iam_binding`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_binding) resource -- `iam_additive`, `iam_additive_members` and `iam_members` configure additive bindings that only manage individual role/member pairs, mapping to the [`google_project_iam_member`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_member) resource -- `iam_policy` which controls the entire IAM policy for the project, where any binding created outside this module (eg in the console) will be removed at each `terraform apply` cycle regardless of the role +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions -The authoritative and additive approaches can be used together, provided different roles are managed by each. The IAM policy is incompatible with the other approaches, and must be used with extreme care. +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. Be mindful about service identity roles when using authoritative IAM, as you might inadvertently remove a role from a [service identity](https://cloud.google.com/iam/docs/service-account-types#google-managed) or default service account. For example, using `roles/editor` with `iam` or `group_iam` will remove the default permissions for the Cloud Services identity. A simple workaround for these scenarios is described below. @@ -107,75 +103,24 @@ module "project" { # tftest modules=1 resources=5 inventory=iam-group.yaml ``` -### Additive IAM - -Additive IAM is typically used where bindings for specific roles are controlled by different modules or in different Terraform stages. One example is when the project is created by one team but a different team manages service account creation for the project, and some of the project-level roles overlap in the two configurations. - -#### Additive IAM by Role - -Additive IAM is supported via the `iam_additive` variable which is keyed by role: +The `iam_bindings` variable behaves like a more verbose version of `iam`, and allows setting binding-level IAM conditions. ```hcl module "project" { - source = "./fabric/modules/project" - name = "project-example" - iam_additive = { - "roles/viewer" = [ - "group:one@example.org", - "group:two@xample.org" - ], - "roles/storage.objectAdmin" = [ - "group:two@example.org" - ], - "roles/owner" = [ - "group:three@example.org" - ], - } -} -# tftest modules=1 resources=5 inventory=iam-additive.yaml -``` - -#### Additive IAM by Principal - -Additive IAM is also supported via the `iam_additive_members` variable which is keyed by principal: - -```hcl -module "project" { - source = "./fabric/modules/project" - name = "project-example" - iam_additive_members = { - "user:one@example.org" = ["roles/owner"] - "user:two@example.org" = ["roles/owner", "roles/editor"] - } - -} -# tftest modules=1 resources=4 inventory=iam-additive-members.yaml -``` - -#### Additive IAM by Binding - -When the above approaches to additive IAM are unworkable due to dynamically generated principals, the `iam_members` variable allows specifying individual role/principal pairs using arbitrary keys. This IAM variable also supports conditions. - -```hcl -module "project" { - source = "./fabric/modules/project" - name = "project-example" - iam_members = { - one-owner = { - member = "user:one@example.org" - role = "roles/owner" - } - two-viewer = { - member = "user:two@example.org" - role = "roles/viewer" - } - two-compute-admin = { - member = "user:two@example.org" - role = "roles/compute.admin" - } - one-delegated-grant = { - member = "user:one@example.org" - role = "roles/resourcemanager.projectIamAdmin" + source = "./fabric/modules/project" + billing_account = "123456-123456-123456" + name = "project-example" + parent = "folders/1234567890" + prefix = "foo" + services = [ + "container.googleapis.com", + "stackdriver.googleapis.com" + ] + iam_bindings = { + "roles/resourcemanager.projectIamAdmin" = { + members = [ + "group:test-admins@example.org" + ] condition = { title = "delegated_network_user_one" expression = <<-END @@ -189,7 +134,30 @@ module "project" { } } } -# tftest modules=1 resources=5 inventory=iam-members.yaml +# tftest modules=1 resources=4 inventory=iam-bindings.yaml +``` + +### Additive IAM + +Additive IAM is typically used where bindings for specific roles are controlled by different modules or in different Terraform stages. One common example is a host project managed by the networking team, and a project factory that manages service projects and needs to assign `roles/networkUser` on the host project. + +The `iam_bindings_additive` variable allows setting individual role/principal binding pairs. Support for IAM conditions is implemented like for `iam_bindings` above. + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "project-1" + services = [ + "compute.googleapis.com" + ] + iam_bindings_additive = { + group-owner = { + member = "group:p1-owners@example.org" + role = "roles/owner" + } + } +} +# tftest modules=1 resources=3 inventory=iam-bindings-additive.yaml ``` ### Service Identities and Authoritative IAM @@ -214,29 +182,6 @@ module "project" { # tftest modules=1 resources=2 ``` -### Using Shortcodes for Service Identities in Additive Iam - -Most Service Identities contains project number in their e-mail address and this prevents additive IAM to work, as these values are not known at moment of execution of `terraform plan` (its not an issue for authoritative IAM). To refer current project Service Identities you may use shortcodes for Service Identities similarly as for `service_identity_iam` when configuring Shared VPC. - -```hcl -module "project" { - source = "./fabric/modules/project" - name = "project-example" - - services = [ - "run.googleapis.com", - "container.googleapis.com", - ] - - iam_additive = { - "roles/editor" = ["cloudservices"] - "roles/vpcaccess.user" = ["cloudrun"] - "roles/container.hostServiceAgentUser" = ["container-engine"] - } -} -# tftest modules=1 resources=6 -``` - ### Service Identities Requiring Manual Iam Grants The module will create service identities at project creation instead of creating of them at the time of first use. This allows granting these service identities roles in other projects, something which is usually necessary in a Shared VPC context. @@ -544,33 +489,6 @@ module "project" { # tftest modules=1 resources=3 inventory=logging-data-access.yaml ``` -While this sets an authoritative policies that has exclusive control of both IAM bindings for all roles and data access log configuration, and should be used with extreme care: - -```hcl -module "project" { - source = "./fabric/modules/project" - name = "my-project" - billing_account = "123456-123456-123456" - parent = "folders/1234567890" - iam_policy = { - "roles/owner" = ["group:org-admins@example.com"] - "roles/resourcemanager.folderAdmin" = ["group:org-admins@example.com"] - "roles/resourcemanager.organizationAdmin" = ["group:org-admins@example.com"] - "roles/resourcemanager.projectCreator" = ["group:org-admins@example.com"] - } - logging_data_access = { - allServices = { - ADMIN_READ = ["group:organization-admins@example.org"] - } - "storage.googleapis.com" = { - DATA_READ = [] - DATA_WRITE = [] - } - } -} -# tftest modules=1 resources=2 inventory=iam-policy.yaml -``` - ## Cloud Kms Encryption Keys The module offers a simple, centralized way to assign `roles/cloudkms.cryptoKeyEncrypterDecrypter` to service identities. @@ -655,7 +573,7 @@ output "compute_robot" { | name | description | resources | |---|---|---| -| [iam.tf](./iam.tf) | Generic and OSLogin-specific IAM bindings and roles. | google_project_iam_binding · google_project_iam_custom_role · google_project_iam_member · google_project_iam_policy | +| [iam.tf](./iam.tf) | Generic and OSLogin-specific IAM bindings and roles. | google_project_iam_binding · google_project_iam_custom_role · google_project_iam_member | | [logging.tf](./logging.tf) | Log sinks and supporting resources. | google_bigquery_dataset_iam_member · google_logging_project_exclusion · google_logging_project_sink · google_project_iam_audit_config · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | | [main.tf](./main.tf) | Module-level locals and resources. | google_compute_project_metadata_item · google_essential_contacts_contact · google_monitoring_monitored_project · google_project · google_project_service · google_resource_manager_lien | | [organization-policies.tf](./organization-policies.tf) | Project-level organization policies. | google_org_policy_policy | @@ -671,42 +589,38 @@ output "compute_robot" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L176) | Project name and id suffix. | string | ✓ | | +| [name](variables.tf#L185) | Project name and id suffix. | string | ✓ | | | [auto_create_network](variables.tf#L17) | Whether to create the default network for the project. | bool | | false | | [billing_account](variables.tf#L23) | Billing account id. | string | | null | -| [contacts](variables.tf#L29) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | -| [custom_roles](variables.tf#L36) | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | -| [default_service_account](variables.tf#L43) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | string | | "keep" | -| [descriptive_name](variables.tf#L49) | Name of the project name. Used for project name instead of `name` variable. | string | | null | -| [group_iam](variables.tf#L55) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | -| [iam](variables.tf#L62) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L69) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive_members](variables.tf#L76) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L82) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [iam_policy](variables.tf#L97) | IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution. | map(list(string)) | | null | -| [labels](variables.tf#L103) | Resource labels. | map(string) | | {} | -| [lien_reason](variables.tf#L110) | If non-empty, creates a project lien with this description. | string | | "" | -| [logging_data_access](variables.tf#L116) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L131) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L138) | Logging sinks to create for this project. | map(object({…})) | | {} | -| [metric_scopes](variables.tf#L169) | List of projects that will act as metric scopes for this project. | list(string) | | [] | -| [org_policies](variables.tf#L181) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | -| [org_policies_data_path](variables.tf#L208) | Path containing org policies in YAML format. | string | | null | -| [oslogin](variables.tf#L214) | Enable OS Login. | bool | | false | -| [oslogin_admins](variables.tf#L220) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | -| [oslogin_users](variables.tf#L228) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | -| [parent](variables.tf#L235) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | -| [prefix](variables.tf#L245) | Optional prefix used to generate project id and name. | string | | null | -| [project_create](variables.tf#L255) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | -| [service_config](variables.tf#L261) | Configure service API activation. | object({…}) | | {…} | -| [service_encryption_key_ids](variables.tf#L273) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | -| [service_perimeter_bridges](variables.tf#L280) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | -| [service_perimeter_standard](variables.tf#L287) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | -| [services](variables.tf#L293) | Service APIs to enable. | list(string) | | [] | -| [shared_vpc_host_config](variables.tf#L299) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | -| [shared_vpc_service_config](variables.tf#L308) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | -| [skip_delete](variables.tf#L330) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | -| [tag_bindings](variables.tf#L336) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | +| [compute_metadata](variables.tf#L29) | Optional compute metadata key/values. Only usable if compute API has been enabled. | map(string) | | {} | +| [contacts](variables.tf#L36) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | +| [custom_roles](variables.tf#L43) | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | +| [default_service_account](variables.tf#L50) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | string | | "keep" | +| [descriptive_name](variables.tf#L63) | Name of the project name. Used for project name instead of `name` variable. | string | | null | +| [group_iam](variables.tf#L69) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L76) | Authoritative IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L83) | Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L97) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [labels](variables.tf#L112) | Resource labels. | map(string) | | {} | +| [lien_reason](variables.tf#L119) | If non-empty, creates a project lien with this description. | string | | null | +| [logging_data_access](variables.tf#L125) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L140) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L147) | Logging sinks to create for this project. | map(object({…})) | | {} | +| [metric_scopes](variables.tf#L178) | List of projects that will act as metric scopes for this project. | list(string) | | [] | +| [org_policies](variables.tf#L190) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L217) | Path containing org policies in YAML format. | string | | null | +| [parent](variables.tf#L223) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | +| [prefix](variables.tf#L233) | Optional prefix used to generate project id and name. | string | | null | +| [project_create](variables.tf#L243) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | +| [service_config](variables.tf#L249) | Configure service API activation. | object({…}) | | {…} | +| [service_encryption_key_ids](variables.tf#L261) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | +| [service_perimeter_bridges](variables.tf#L268) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | +| [service_perimeter_standard](variables.tf#L275) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | +| [services](variables.tf#L281) | Service APIs to enable. | list(string) | | [] | +| [shared_vpc_host_config](variables.tf#L287) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | +| [shared_vpc_service_config](variables.tf#L296) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | +| [skip_delete](variables.tf#L318) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | +| [tag_bindings](variables.tf#L324) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | ## Outputs diff --git a/modules/project/iam.tf b/modules/project/iam.tf index 0918b552..16f187d6 100644 --- a/modules/project/iam.tf +++ b/modules/project/iam.tf @@ -18,8 +18,6 @@ # IAM notes: # - external users need to have accepted the invitation email to join -# - oslogin roles also require role to list instances -# - additive (non-authoritative) roles might fail due to dynamic values locals { _group_iam_roles = distinct(flatten(values(var.group_iam))) @@ -28,16 +26,6 @@ locals { for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null ] } - _iam_additive_pairs = flatten([ - for role, members in var.iam_additive : [ - for member in members : { role = role, member = member } - ] - ]) - _iam_additive_member_pairs = flatten([ - for member, roles in var.iam_additive_members : [ - for role in roles : { role = role, member = member } - ] - ]) iam = { for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : role => concat( @@ -45,21 +33,6 @@ locals { try(local._group_iam[role], []) ) } - iam_additive = { - for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) : - "${pair.role}-${pair.member}" => { - role = pair.role - member = ( - pair.member == "cloudservices" - ? "serviceAccount:${local.service_account_cloud_services}" - : pair.member == "default-compute" - ? "serviceAccount:${local.service_accounts_default.compute}" - : pair.member == "default-gae" - ? "serviceAccount:${local.service_accounts_default.gae}" - : try("serviceAccount:${local.service_accounts_robots[pair.member]}", pair.member) - ) - } - } } resource "google_project_iam_custom_role" "roles" { @@ -82,23 +55,27 @@ resource "google_project_iam_binding" "authoritative" { ] } -resource "google_project_iam_member" "additive" { - for_each = ( - length(var.iam_additive) + length(var.iam_additive_members) > 0 - ? local.iam_additive - : {} - ) - project = local.project.project_id - role = each.value.role - member = each.value.member +resource "google_project_iam_binding" "bindings" { + for_each = var.iam_bindings + project = local.project.project_id + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } depends_on = [ google_project_service.project_services, google_project_iam_custom_role.roles ] } -resource "google_project_iam_member" "members" { - for_each = var.iam_members +resource "google_project_iam_member" "bindings" { + for_each = var.iam_bindings_additive project = local.project.project_id role = each.value.role member = each.value.member @@ -115,62 +92,3 @@ resource "google_project_iam_member" "members" { google_project_iam_custom_role.roles ] } - -resource "google_project_iam_member" "oslogin_iam_serviceaccountuser" { - for_each = var.oslogin ? toset(distinct(concat(var.oslogin_admins, var.oslogin_users))) : toset([]) - project = local.project.project_id - role = "roles/iam.serviceAccountUser" - member = each.value -} - -resource "google_project_iam_member" "oslogin_compute_viewer" { - for_each = var.oslogin ? toset(distinct(concat(var.oslogin_admins, var.oslogin_users))) : toset([]) - project = local.project.project_id - role = "roles/compute.viewer" - member = each.value -} - -resource "google_project_iam_member" "oslogin_admins" { - for_each = var.oslogin ? toset(var.oslogin_admins) : toset([]) - project = local.project.project_id - role = "roles/compute.osAdminLogin" - member = each.value -} - -resource "google_project_iam_member" "oslogin_users" { - for_each = var.oslogin ? toset(var.oslogin_users) : toset([]) - project = local.project.project_id - role = "roles/compute.osLogin" - member = each.value -} - -resource "google_project_iam_policy" "authoritative" { - count = var.iam_policy != null ? 1 : 0 - project = local.project.project_id - policy_data = data.google_iam_policy.authoritative.0.policy_data -} - -data "google_iam_policy" "authoritative" { - count = var.iam_policy != null ? 1 : 0 - dynamic "binding" { - for_each = try(var.iam_policy, {}) - content { - role = binding.key - members = binding.value - } - } - dynamic "audit_config" { - for_each = var.logging_data_access - content { - service = audit_config.key - dynamic "audit_log_configs" { - for_each = audit_config.value - iterator = config - content { - log_type = config.key - exempted_members = config.value - } - } - } - } -} diff --git a/modules/project/logging.tf b/modules/project/logging.tf index e80254da..0181f04f 100644 --- a/modules/project/logging.tf +++ b/modules/project/logging.tf @@ -27,11 +27,9 @@ locals { } resource "google_project_iam_audit_config" "default" { - for_each = ( - var.iam_policy == null ? var.logging_data_access : {} - ) - project = local.project.project_id - service = each.key + for_each = var.logging_data_access + project = local.project.project_id + service = each.key dynamic "audit_log_config" { for_each = each.value iterator = config @@ -70,7 +68,8 @@ resource "google_logging_project_sink" "sink" { depends_on = [ google_project_iam_binding.authoritative, - google_project_iam_member.additive + google_project_iam_binding.bindings, + google_project_iam_member.bindings ] } diff --git a/modules/project/main.tf b/modules/project/main.tf index 9f0dff49..547f1aa8 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -61,17 +61,18 @@ resource "google_project_service" "project_services" { disable_dependent_services = var.service_config.disable_dependent_services } -resource "google_compute_project_metadata_item" "oslogin_meta" { - count = var.oslogin ? 1 : 0 - project = local.project.project_id - key = "enable-oslogin" - value = "TRUE" - # depend on services or it will fail on destroy +resource "google_compute_project_metadata_item" "default" { + for_each = ( + contains(var.services, "compute.googleapis.com") ? var.compute_metadata : {} + ) + project = local.project.project_id + key = each.key + value = each.value depends_on = [google_project_service.project_services] } resource "google_resource_manager_lien" "lien" { - count = var.lien_reason != "" ? 1 : 0 + count = var.lien_reason != null ? 1 : 0 parent = "projects/${local.project.number}" restrictions = ["resourcemanager.projects.delete"] origin = "created-by-terraform" diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 18f55886..2824fcf3 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -26,6 +26,13 @@ variable "billing_account" { default = null } +variable "compute_metadata" { + description = "Optional compute metadata key/values. Only usable if compute API has been enabled." + type = map(string) + nullable = false + default = {} +} + variable "contacts" { description = "List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES." type = map(list(string)) @@ -44,6 +51,13 @@ variable "default_service_account" { description = "Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`." default = "keep" type = string + validation { + condition = ( + var.default_service_account == null || + contains(["delete", "deprivilege", "disable", "keep"], var.default_service_account) + ) + error_message = "Only `delete`, `deprivilege`, `disable`, or `keep` are supported." + } } variable "descriptive_name" { @@ -60,27 +74,28 @@ variable "group_iam" { } variable "iam" { - description = "IAM bindings in {ROLE => [MEMBERS]} format." + description = "Authoritative IAM bindings in {ROLE => [MEMBERS]} format." type = map(list(string)) default = {} nullable = false } -variable "iam_additive" { - description = "IAM additive bindings in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} - nullable = false +variable "iam_bindings" { + description = "Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_additive_members" { - description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values." - type = map(list(string)) - default = {} -} - -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string @@ -94,12 +109,6 @@ variable "iam_members" { default = {} } -variable "iam_policy" { - description = "IAM authoritative policy in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared, use with extreme caution." - type = map(list(string)) - default = null -} - variable "labels" { description = "Resource labels." type = map(string) @@ -110,7 +119,7 @@ variable "labels" { variable "lien_reason" { description = "If non-empty, creates a project lien with this description." type = string - default = "" + default = null } variable "logging_data_access" { @@ -211,27 +220,6 @@ variable "org_policies_data_path" { default = null } -variable "oslogin" { - description = "Enable OS Login." - type = bool - default = false -} - -variable "oslogin_admins" { - description = "List of IAM-style identities that will be granted roles necessary for OS Login administrators." - type = list(string) - default = [] - nullable = false - -} - -variable "oslogin_users" { - description = "List of IAM-style identities that will be granted roles necessary for OS Login users." - type = list(string) - default = [] - nullable = false -} - variable "parent" { description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format." type = string diff --git a/modules/source-repository/README.md b/modules/source-repository/README.md index 01fff2d8..a62013fa 100644 --- a/modules/source-repository/README.md +++ b/modules/source-repository/README.md @@ -23,7 +23,7 @@ module "repo" { iam = { "roles/source.reader" = ["user:foo@example.com"] } - iam_members = { + iam_bindings_additive = { am1-reader = { member = "user:am1@example.com" role = "roles/source.reader" @@ -75,14 +75,13 @@ module "repo" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L59) | Repository name. | string | ✓ | | -| [project_id](variables.tf#L64) | Project used for resources. | string | ✓ | | +| [name](variables.tf#L60) | Repository name. | string | ✓ | | +| [project_id](variables.tf#L65) | Project used for resources. | string | ✓ | | | [group_iam](variables.tf#L17) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L24) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L31) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive_members](variables.tf#L38) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | -| [iam_members](variables.tf#L44) | Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop. | map(object({…})) | | {} | -| [triggers](variables.tf#L69) | Cloud Build triggers. | map(object({…})) | | {} | +| [iam_bindings](variables.tf#L31) | Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L45) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [triggers](variables.tf#L70) | Cloud Build triggers. | map(object({…})) | | {} | ## Outputs diff --git a/modules/source-repository/iam.tf b/modules/source-repository/iam.tf index 75d00fd1..be0cf688 100644 --- a/modules/source-repository/iam.tf +++ b/modules/source-repository/iam.tf @@ -23,16 +23,6 @@ locals { for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null ] } - _iam_additive_pairs = flatten([ - for role, members in var.iam_additive : [ - for member in members : { role = role, member = member } - ] - ]) - _iam_additive_member_pairs = flatten([ - for member, roles in var.iam_additive_members : [ - for role in roles : { role = role, member = member } - ] - ]) iam = { for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : role => concat( @@ -40,10 +30,6 @@ locals { try(local._group_iam[role], []) ) } - iam_additive = { - for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) : - "${pair.role}-${pair.member}" => pair - } } resource "google_sourcerepo_repository_iam_binding" "authoritative" { @@ -54,20 +40,24 @@ resource "google_sourcerepo_repository_iam_binding" "authoritative" { members = each.value } -resource "google_sourcerepo_repository_iam_member" "additive" { - for_each = ( - length(var.iam_additive) + length(var.iam_additive_members) > 0 - ? local.iam_additive - : {} - ) +resource "google_sourcerepo_repository_iam_binding" "bindings" { + for_each = var.iam_bindings project = var.project_id repository = google_sourcerepo_repository.default.name - role = each.value.role - member = each.value.member + role = each.key + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } } -resource "google_sourcerepo_repository_iam_member" "members" { - for_each = var.iam_members +resource "google_sourcerepo_repository_iam_member" "bindings" { + for_each = var.iam_bindings_additive project = var.project_id repository = google_sourcerepo_repository.default.name role = each.value.role diff --git a/modules/source-repository/variables.tf b/modules/source-repository/variables.tf index 61975fb9..ce1c34e7 100644 --- a/modules/source-repository/variables.tf +++ b/modules/source-repository/variables.tf @@ -28,21 +28,22 @@ variable "iam" { nullable = false } -variable "iam_additive" { - description = "IAM additive bindings in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} - nullable = false +variable "iam_bindings" { + description = "Authoritative IAM bindings in {ROLE => {members = [], condition = {}}}." + type = map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} } -variable "iam_additive_members" { - description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values." - type = map(list(string)) - default = {} -} - -variable "iam_members" { - description = "Individual additive IAM bindings, use this when iam_additive does not work due to dynamic resources. Keys are arbitrary and only used for the internal loop." +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." type = map(object({ member = string role = string diff --git a/tests/blueprints/factories/project_factory/examples/example.yaml b/tests/blueprints/factories/project_factory/examples/example.yaml index 82c6dd53..5927caed 100644 --- a/tests/blueprints/factories/project_factory/examples/example.yaml +++ b/tests/blueprints/factories/project_factory/examples/example.yaml @@ -12,244 +12,101 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This is one of the few modules where it actually makes sense to be -# very verbose with values - values: - module.projects["project"].google_compute_subnetwork_iam_member.default["dev-default-ew1:serviceAccount:my-service-account"]: - condition: [] - member: serviceAccount:my-service-account - project: fast-dev-net-spoke-0 - region: europe-west1 - role: roles/compute.networkUser - subnetwork: projects/fast-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1 - module.projects["project"].google_compute_subnetwork_iam_member.default["dev-default-ew1:user:foobar@example.com"]: - condition: [] - member: user:foobar@example.com - project: fast-dev-net-spoke-0 - region: europe-west1 - role: roles/compute.networkUser - subnetwork: projects/fast-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1 - module.projects["project"].module.billing-alert["1"].google_billing_budget.budget: - all_updates_rule: - - disable_default_iam_recipients: false - pubsub_topic: null - schema_version: '1.0' - amount: - - last_period_amount: null - specified_amount: - - nanos: null - units: '10' - billing_account: 012345-67890A-BCDEF0 - budget_filter: - - calendar_period: null - credit_types_treatment: INCLUDE_ALL_CREDITS - custom_period: [] - display_name: test1-project budget - threshold_rules: - - spend_basis: CURRENT_SPEND - threshold_percent: 0.5 - - spend_basis: CURRENT_SPEND - threshold_percent: 0.8 - module.projects["project"].module.billing-alert["1"].google_monitoring_notification_channel.email_channels["team-a-contacts@example.com"]: - display_name: test1-project budget budget email notification (team-a-contacts@example.com) - labels: - email_address: team-a-contacts@example.com - project: test1-project - sensitive_labels: [] - type: email - module.projects["project"].module.billing-alert["1"].google_monitoring_notification_channel.email_channels["team-contacts@example.com"]: - display_name: test1-project budget budget email notification (team-contacts@example.com) - labels: - email_address: team-contacts@example.com - project: test1-project - sensitive_labels: [] - type: email - module.projects["project"].module.dns["ipsum"].google_dns_managed_zone.dns_managed_zone[0]: - dns_name: ipsum.dev.example.org - name: ipsum - private_visibility_config: - - gke_clusters: [] - networks: - - network_url: projects/foo/networks/bar - project: fast-dev-net-spoke-0 - visibility: private - module.projects["project"].module.dns["lorem"].google_dns_managed_zone.dns_managed_zone[0]: - dns_name: lorem.dev.example.org - name: lorem - private_visibility_config: - - gke_clusters: [] - networks: - - network_url: projects/foo/networks/bar - project: fast-dev-net-spoke-0 - module.projects["project"].module.project.google_compute_shared_vpc_service_project.shared_vpc_service[0]: - host_project: fast-dev-net-spoke-0 - service_project: test1-project - module.projects["project"].module.project.google_essential_contacts_contact.contact["team-a-contacts@example.com"]: - email: team-a-contacts@example.com + module.project-factory.module.projects["prj-app-1"].data.google_storage_project_service_account.gcs_sa[0]: + project: test-pf-prj-app-1 + user_project: null + module.project-factory.module.projects["prj-app-1"].google_essential_contacts_contact.contact["admin@example.com"]: + email: admin@example.com language_tag: en notification_category_subscriptions: - ALL - parent: projects/test1-project - module.projects["project"].module.project.google_essential_contacts_contact.contact["team-contacts@example.com"]: - email: team-contacts@example.com - language_tag: en - notification_category_subscriptions: - - ALL - parent: projects/test1-project - module.projects["project"].module.project.google_org_policy_policy.default["compute.disableGuestAttributesAccess"]: - name: projects/test1-project/policies/compute.disableGuestAttributesAccess - parent: projects/test1-project - spec: - - inherit_from_parent: null - reset: null - rules: - - allow_all: null - condition: [] - deny_all: null - enforce: 'TRUE' - values: [] - module.projects["project"].module.project.google_org_policy_policy.default["compute.trustedImageProjects"]: - name: projects/test1-project/policies/compute.trustedImageProjects - parent: projects/test1-project - spec: - - inherit_from_parent: null - reset: null - rules: - - allow_all: null - condition: [] - deny_all: null - enforce: null - values: - - allowed_values: - - projects/fast-dev-iac-core-0 - denied_values: null - module.projects["project"].module.project.google_org_policy_policy.default["compute.vmExternalIpAccess"]: - name: projects/test1-project/policies/compute.vmExternalIpAccess - parent: projects/test1-project - spec: - - inherit_from_parent: null - reset: null - rules: - - allow_all: null - condition: [] - deny_all: 'TRUE' - enforce: null - values: [] - module.projects["project"].module.project.google_project.project[0]: + parent: projects/test-pf-prj-app-1 + timeouts: null + ? module.project-factory.module.projects["prj-app-1"].google_kms_crypto_key_iam_member.service_identity_cmek["compute.projects/kms-central-prj/locations/europe-west3/keyRings/my-keyring/cryptoKeys/europe3-gce"] + : condition: [] + crypto_key_id: projects/kms-central-prj/locations/europe-west3/keyRings/my-keyring/cryptoKeys/europe3-gce + role: roles/cloudkms.cryptoKeyEncrypterDecrypter + module.project-factory.module.projects["prj-app-1"].google_project.project[0]: auto_create_network: false billing_account: 012345-67890A-BCDEF0 - folder_id: 012345678901 + folder_id: null labels: - application: example-app - costcenter: apps - department: accounting - environment: dev - foo: bar - name: test1-project + app: app-1 + team: foo + name: test-pf-prj-app-1 org_id: null - project_id: test1-project + project_id: test-pf-prj-app-1 skip_delete: false - module.projects["project"].module.project.google_project_iam_binding.authoritative["roles/compute.admin"]: - condition: [] - project: test1-project - role: roles/compute.admin - module.projects["project"].module.project.google_project_iam_binding.authoritative["roles/compute.adminv1"]: - condition: [] - project: test1-project - role: roles/compute.adminv1 - module.projects["project"].module.project.google_project_iam_binding.authoritative["roles/storage.objectViewer"]: - condition: [] - project: test1-project - role: roles/storage.objectViewer - module.projects["project"].module.project.google_project_iam_member.shared_vpc_host_robots["roles/compute.networkUser:cloudservices"]: - condition: [] - project: fast-dev-net-spoke-0 - role: roles/compute.networkUser - module.projects["project"].module.project.google_project_iam_member.shared_vpc_host_robots["roles/compute.securityAdmin:container-engine"]: - condition: [] - project: fast-dev-net-spoke-0 - role: roles/compute.securityAdmin - module.projects["project"].module.project.google_project_iam_member.shared_vpc_host_robots["roles/compute.networkUser:dataproc"]: - project: fast-dev-net-spoke-0 - role: roles/compute.networkUser - module.projects["project"].module.project.google_kms_crypto_key_iam_member.service_identity_cmek["compute.key1"]: - condition: [] - crypto_key_id: key1 - role: roles/cloudkms.cryptoKeyEncrypterDecrypter - module.projects["project"].module.project.google_kms_crypto_key_iam_member.service_identity_cmek["compute.key2"]: - condition: [] - crypto_key_id: key2 - role: roles/cloudkms.cryptoKeyEncrypterDecrypter - module.projects["project"].module.project.google_kms_crypto_key_iam_member.service_identity_cmek["storage.key1"]: - condition: [] - crypto_key_id: key1 - role: roles/cloudkms.cryptoKeyEncrypterDecrypter - module.projects["project"].module.project.google_kms_crypto_key_iam_member.service_identity_cmek["storage.key2"]: - condition: [] - crypto_key_id: key2 - role: roles/cloudkms.cryptoKeyEncrypterDecrypter - module.projects["project"].module.project.google_project_service.project_services["billingbudgets.googleapis.com"]: + timeouts: null + module.project-factory.module.projects["prj-app-1"].google_project_service.project_services["stackdriver.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test1-project - service: billingbudgets.googleapis.com - module.projects["project"].module.project.google_project_service.project_services["compute.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test1-project - service: compute.googleapis.com - module.projects["project"].module.project.google_project_service.project_services["container.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test1-project - service: container.googleapis.com - module.projects["project"].module.project.google_project_service.project_services["dns.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test1-project - service: dns.googleapis.com - module.projects["project"].module.project.google_project_service.project_services["essentialcontacts.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test1-project - service: essentialcontacts.googleapis.com - module.projects["project"].module.project.google_project_service.project_services["orgpolicy.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test1-project - service: orgpolicy.googleapis.com - module.projects["project"].module.project.google_project_service.project_services["stackdriver.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test1-project + project: test-pf-prj-app-1 service: stackdriver.googleapis.com - module.projects["project"].module.project.google_project_service.project_services["storage.googleapis.com"]: + timeouts: null + module.project-factory.module.projects["prj-app-1"].google_project_service.project_services["storage.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test1-project + project: test-pf-prj-app-1 service: storage.googleapis.com - module.projects["project"].module.service-accounts["another-service-account"].google_service_account.service_account[0]: - account_id: another-service-account + timeouts: null + module.project-factory.module.projects["prj-app-2"].google_essential_contacts_contact.contact["admin@example.com"]: + email: admin@example.com + language_tag: en + notification_category_subscriptions: + - ALL + parent: projects/test-pf-prj-app-2 + timeouts: null + module.project-factory.module.projects["prj-app-2"].google_project.project[0]: + auto_create_network: false + billing_account: 012345-67890A-ABCDEF + folder_id: null + labels: + app: app-1 + team: foo + name: test-pf-prj-app-2 + org_id: null + project_id: test-pf-prj-app-2 + skip_delete: false + timeouts: null + module.project-factory.module.projects["prj-app-2"].google_project_service.project_services["stackdriver.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-prj-app-2 + service: stackdriver.googleapis.com + timeouts: null + module.project-factory.module.service-accounts["prj-app-1-app-1-be"].google_service_account.service_account[0]: + account_id: app-1-be + description: null + disabled: false display_name: Terraform-managed. - project: test1-project - module.projects["project"].module.service-accounts["my-service-account"].google_service_account.service_account[0]: - account_id: my-service-account + project: test-pf-prj-app-1 + timeouts: null + module.project-factory.module.service-accounts["prj-app-1-app-1-fe"].google_service_account.service_account[0]: + account_id: app-1-fe + description: null + disabled: false display_name: Terraform-managed. - project: test1-project + project: test-pf-prj-app-1 + timeouts: null + module.project-factory.module.service-accounts["prj-app-2-app-2-be"].google_service_account.service_account[0]: + account_id: app-2-be + description: null + disabled: false + display_name: Terraform-managed. + project: test-pf-prj-app-2 + timeouts: null counts: - google_billing_budget: 1 - google_compute_shared_vpc_service_project: 1 - google_compute_subnetwork_iam_member: 2 - google_dns_managed_zone: 2 google_essential_contacts_contact: 2 - google_monitoring_notification_channel: 2 - google_org_policy_policy: 3 - google_project: 1 - google_project_iam_binding: 3 - google_project_iam_member: 4 - google_project_service: 8 - google_service_account: 2 + google_kms_crypto_key_iam_member: 1 + google_project: 2 + google_project_service: 3 + google_service_account: 3 google_storage_project_service_account: 1 - google_kms_crypto_key_iam_member: 4 + modules: 6 + resources: 12 + +outputs: {} + diff --git a/tests/examples/conftest.py b/tests/examples/conftest.py index 4d3d85ee..55702618 100644 --- a/tests/examples/conftest.py +++ b/tests/examples/conftest.py @@ -66,7 +66,7 @@ def pytest_generate_tests(metafunc): name = f'{path}:{last_header}' if index > 1: name += f' {index}' - ids.append(name) + ids.append(f'{path}:{last_header}:{index}') examples.append(Example(name, code, path, files[last_header])) elif isinstance(child, marko.block.Heading): last_header = child.children[0].children diff --git a/tests/fast/stages/s0_bootstrap/simple.yaml b/tests/fast/stages/s0_bootstrap/simple.yaml index d862c2d0..b73cda2b 100644 --- a/tests/fast/stages/s0_bootstrap/simple.yaml +++ b/tests/fast/stages/s0_bootstrap/simple.yaml @@ -18,7 +18,7 @@ counts: google_logging_organization_sink: 2 google_organization_iam_binding: 20 google_organization_iam_custom_role: 3 - google_organization_iam_member: 16 + google_organization_iam_member: 17 google_project: 3 google_project_iam_binding: 9 google_project_iam_member: 3 diff --git a/tests/fast/stages/s3_project_factory/common.tfvars b/tests/fast/stages/s3_project_factory/common.tfvars index d3f8c6f9..d26b3e68 100644 --- a/tests/fast/stages/s3_project_factory/common.tfvars +++ b/tests/fast/stages/s3_project_factory/common.tfvars @@ -1,7 +1,7 @@ -data_dir = "../../../../tests/fast/stages/s3_project_factory/data/projects/" -defaults_file = "../../../../tests/fast/stages/s3_project_factory/data/defaults.yaml" -prefix = "test" -environment_dns_zone = "dev" +factory_data = { + data_path = "../../../../tests/fast/stages/s3_project_factory/data/projects/" +} +prefix = "test" billing_account = { id = "000000-111111-222222" } diff --git a/tests/fast/stages/s3_project_factory/data/defaults.yaml b/tests/fast/stages/s3_project_factory/data/defaults.yaml deleted file mode 100644 index b050583f..00000000 --- a/tests/fast/stages/s3_project_factory/data/defaults.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -billing_account_id: 012345-67890A-BCDEF0 - -# [opt] Setup for billing alerts -billing_alert: - amount: 1000 - thresholds: - current: [0.5, 0.8] - forecasted: [0.5, 0.8] - credit_treatment: INCLUDE_ALL_CREDITS - -# [opt] Contacts for billing alerts and important notifications -essential_contacts: ["team-contacts@example.com"] - -# [opt] Labels set for all projects -labels: - environment: prod - department: accounting - application: example-app - foo: bar - -# [opt] Additional notification channels for billing -notification_channels: [] diff --git a/tests/fast/stages/s3_project_factory/data/projects/project.yaml b/tests/fast/stages/s3_project_factory/data/projects/project.yaml index 90354a2a..18b5cdb4 100644 --- a/tests/fast/stages/s3_project_factory/data/projects/project.yaml +++ b/tests/fast/stages/s3_project_factory/data/projects/project.yaml @@ -12,100 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [opt] Billing account id - overrides default if set -billing_account_id: 012345-67890A-BCDEF0 - -# [opt] Billing alerts config - overrides default if set -billing_alert: - amount: 10 - thresholds: - current: - - 0.5 - - 0.8 - forecasted: [] - credit_treatment: INCLUDE_ALL_CREDITS - -# [opt] DNS zones to be created as children of the environment_dns_zone defined in defaults -dns_zones: - - lorem - - ipsum - -# [opt] Contacts for billing alerts and important notifications -essential_contacts: - - team-a-contacts@example.com - -# Folder the project will be created as children of -folder_id: folders/012345678901 - -# [opt] Authoritative IAM bindings in group => [roles] format -group_iam: - test-team-foobar@fast-lab-0.gcp-pso-italy.net: - - roles/compute.admin - -# [opt] Authoritative IAM bindings in role => [principals] format -# Generally used to grant roles to service accounts external to the project -iam: - roles/compute.admin: - - serviceAccount:service-account - -# [opt] Service robots and keys they will be assigned as cryptoKeyEncrypterDecrypter -# in service => [keys] format -kms_service_agents: - compute: [key1, key2] - storage: [key1, key2] - -# [opt] Labels for the project - merged with the ones defined in defaults -labels: - environment: prod - -# [opt] Org policy overrides defined at project level -org_policies: - compute.disableGuestAttributesAccess: - rules: - - enforce: true - compute.trustedImageProjects: - rules: - - allow: - values: - - projects/fast-prod-iac-core-0 - -# [opt] Service account to create for the project and their roles on the project -# in name => [roles] format -service_accounts: - another-service-account: - - roles/compute.admin - my-service-account: - - roles/compute.admin - -# [opt] APIs to enable on the project. +parent_id: folders/012345678901 services: - storage.googleapis.com - stackdriver.googleapis.com - compute.googleapis.com - -# [opt] Roles to assign to the robots service accounts in robot => [roles] format -services_iam: - compute: - - roles/storage.objectViewer - - # [opt] VPC setup. - # If set enables the `compute.googleapis.com` service and configures - # service project attachment -vpc: - # [opt] If set, enables the container API - gke_setup: - # Grants "roles/container.hostServiceAgentUser" to the container robot if set - enable_host_service_agent: false - - # Grants "roles/compute.securityAdmin" to the container robot if set - enable_security_admin: true - - # Host project the project will be service project of - host_project: fast-prod-net-spoke-0 - - # [opt] Subnets in the host project where principals will be granted networkUser - # in region/subnet-name => [principals] - subnets_iam: - europe-west1/prod-default-ew1: - - user:foobar@example.com - - serviceAccount:service-account1 diff --git a/tests/modules/dataplex_datascan/examples/datascan_iam.yaml b/tests/modules/dataplex_datascan/examples/datascan_iam.yaml index 28885cfe..35e97f9e 100644 --- a/tests/modules/dataplex_datascan/examples/datascan_iam.yaml +++ b/tests/modules/dataplex_datascan/examples/datascan_iam.yaml @@ -58,7 +58,7 @@ values: - group:user-group@example.com project: my-project-name role: roles/dataplex.dataScanViewer - module.dataplex-datascan.google_dataplex_datascan_iam_member.members["am1-viewer"]: + module.dataplex-datascan.google_dataplex_datascan_iam_member.bindings["am1-viewer"]: condition: [] data_scan_id: test-datascan location: us-central1 diff --git a/tests/modules/folder/examples/iam-policy.yaml b/tests/modules/folder/examples/iam-policy.yaml deleted file mode 100644 index 5ed33ab4..00000000 --- a/tests/modules/folder/examples/iam-policy.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.folder.google_folder.folder[0]: - display_name: my-folder - parent: folders/657104291943 - timeouts: null - module.folder.google_folder_iam_policy.authoritative[0]: - policy_data: '{"auditConfigs":[{"auditLogConfigs":[{"exemptedMembers":["group:organization-admins@example.org"],"logType":"ADMIN_READ"}],"service":"allServices"},{"auditLogConfigs":[{"logType":"DATA_WRITE"},{"logType":"DATA_READ"}],"service":"storage.googleapis.com"}],"bindings":[{"members":["group:org-admins@example.com"],"role":"roles/owner"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.folderAdmin"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.organizationAdmin"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.projectCreator"}]}' - -counts: - google_folder: 1 - google_folder_iam_policy: 1 - modules: 1 - resources: 2 diff --git a/tests/modules/folder/examples/iam.yaml b/tests/modules/folder/examples/iam.yaml index 541fbf86..39fa5652 100644 --- a/tests/modules/folder/examples/iam.yaml +++ b/tests/modules/folder/examples/iam.yaml @@ -33,27 +33,7 @@ values: members: - group:cloud-owners@example.org role: roles/resourcemanager.projectCreator - module.folder.google_folder_iam_member.additive["roles/compute.admin-user:a1@example.org"]: - condition: [] - member: user:a1@example.org - role: roles/compute.admin - module.folder.google_folder_iam_member.additive["roles/compute.admin-user:a2@example.org"]: - condition: [] - member: user:a2@example.org - role: roles/compute.admin - module.folder.google_folder_iam_member.additive["roles/compute.viewer-user:a2@example.org"]: - condition: [] - member: user:a2@example.org - role: roles/compute.viewer - module.folder.google_folder_iam_member.additive["roles/storage.admin-user:am1@example.org"]: - condition: [] - member: user:am1@example.org - role: roles/storage.admin - module.folder.google_folder_iam_member.additive["roles/storage.objectViewer-user:am2@example.org"]: - condition: [] - member: user:am2@example.org - role: roles/storage.objectViewer - module.folder.google_folder_iam_member.members["am1-storage-admin"]: + module.folder.google_folder_iam_member.bindings["am1-storage-admin"]: condition: [] member: user:am1@example.org role: roles/storage.admin @@ -61,8 +41,9 @@ values: counts: google_folder: 1 google_folder_iam_binding: 3 - google_folder_iam_member: 6 + google_folder_iam_member: 1 modules: 1 - resources: 10 + resources: 5 outputs: {} + diff --git a/tests/modules/iam_service_account/examples/basic.yaml b/tests/modules/iam_service_account/examples/basic.yaml index 4acc5851..2a1f4efc 100644 --- a/tests/modules/iam_service_account/examples/basic.yaml +++ b/tests/modules/iam_service_account/examples/basic.yaml @@ -27,7 +27,7 @@ values: display_name: Terraform-managed. project: myproject timeouts: null - module.myproject-default-service-accounts.google_service_account_iam_binding.roles["roles/iam.serviceAccountUser"]: + module.myproject-default-service-accounts.google_service_account_iam_binding.authoritative["roles/iam.serviceAccountUser"]: condition: [] members: - user:foo@example.com diff --git a/tests/modules/net_vpc/examples/factory.yaml b/tests/modules/net_vpc/examples/factory.yaml index 21ee1565..fb348397 100644 --- a/tests/modules/net_vpc/examples/factory.yaml +++ b/tests/modules/net_vpc/examples/factory.yaml @@ -14,12 +14,66 @@ values: module.vpc.google_compute_network.network[0]: + auto_create_subnetworks: false + delete_default_routes_on_create: false + description: Terraform-managed. + enable_ula_internal_ipv6: null name: my-network + network_firewall_policy_enforcement_order: AFTER_CLASSIC_FIREWALL project: my-project routing_mode: GLOBAL + timeouts: null + module.vpc.google_compute_route.gateway["private-googleapis"]: + description: Terraform-managed. + dest_range: 199.36.153.8/30 + name: my-network-private-googleapis + next_hop_gateway: default-internet-gateway + next_hop_ilb: null + next_hop_instance: null + next_hop_vpn_tunnel: null + priority: 1000 + project: my-project + tags: null + timeouts: null + module.vpc.google_compute_route.gateway["restricted-googleapis"]: + description: Terraform-managed. + dest_range: 199.36.153.4/30 + name: my-network-restricted-googleapis + next_hop_gateway: default-internet-gateway + next_hop_ilb: null + next_hop_instance: null + next_hop_vpn_tunnel: null + priority: 1000 + project: my-project + tags: null + timeouts: null + module.vpc.google_compute_subnetwork.proxy_only["europe-west4/subnet-proxy"]: + description: Terraform-managed proxy-only subnet for Regional HTTPS or Internal + HTTPS LB. + ip_cidr_range: 10.1.0.0/24 + ipv6_access_type: null + log_config: [] + name: subnet-proxy + project: my-project + purpose: REGIONAL_MANAGED_PROXY + region: europe-west4 + role: ACTIVE + timeouts: null + module.vpc.google_compute_subnetwork.psc["europe-west4/subnet-psc"]: + description: Terraform-managed subnet for Private Service Connect (PSC NAT). + ip_cidr_range: 10.2.0.0/24 + ipv6_access_type: null + log_config: [] + name: subnet-psc + project: my-project + purpose: PRIVATE_SERVICE_CONNECT + region: europe-west4 + role: null + timeouts: null module.vpc.google_compute_subnetwork.subnetwork["europe-west1/subnet-detailed"]: description: Sample description ip_cidr_range: 10.0.0.0/24 + ipv6_access_type: null log_config: - aggregation_interval: INTERVAL_5_SEC filter_expr: 'true' @@ -34,9 +88,11 @@ values: secondary_ip_range: - ip_cidr_range: 192.168.0.0/24 range_name: secondary-range-a + timeouts: null module.vpc.google_compute_subnetwork.subnetwork["europe-west4/simple"]: description: Terraform-managed. ip_cidr_range: 10.0.1.0/24 + ipv6_access_type: null log_config: [] name: simple private_ip_google_access: true @@ -44,9 +100,11 @@ values: region: europe-west4 role: null secondary_ip_range: [] + timeouts: null module.vpc.google_compute_subnetwork.subnetwork["europe-west8/simple"]: description: Terraform-managed. ip_cidr_range: 10.0.2.0/24 + ipv6_access_type: null log_config: [] name: simple private_ip_google_access: true @@ -54,7 +112,8 @@ values: region: europe-west8 role: null secondary_ip_range: [] - module.vpc.google_compute_subnetwork_iam_binding.binding["europe-west1/subnet-detailed.roles/compute.networkUser"]: + timeouts: null + module.vpc.google_compute_subnetwork_iam_binding.authoritative["europe-west1/subnet-detailed.roles/compute.networkUser"]: condition: [] members: - group:lorem@example.com @@ -64,16 +123,13 @@ values: region: europe-west1 role: roles/compute.networkUser subnetwork: subnet-detailed - module.vpc.google_compute_subnetwork.proxy_only["europe-west4/subnet-proxy"]: - region: europe-west4 - ip_cidr_range: 10.1.0.0/24 - purpose: REGIONAL_MANAGED_PROXY - module.vpc.google_compute_subnetwork.psc["europe-west4/subnet-psc"]: - region: europe-west4 - ip_cidr_range: 10.2.0.0/24 - purpose: PRIVATE_SERVICE_CONNECT counts: google_compute_network: 1 + google_compute_route: 2 google_compute_subnetwork: 5 google_compute_subnetwork_iam_binding: 1 + modules: 1 + resources: 9 + +outputs: {} diff --git a/tests/modules/net_vpc/examples/shared-vpc.yaml b/tests/modules/net_vpc/examples/shared-vpc.yaml index 6467fd38..5f1b9d5a 100644 --- a/tests/modules/net_vpc/examples/shared-vpc.yaml +++ b/tests/modules/net_vpc/examples/shared-vpc.yaml @@ -30,7 +30,7 @@ values: range_name: pods - ip_cidr_range: 192.168.0.0/24 range_name: services - module.vpc-host.google_compute_subnetwork_iam_binding.binding["europe-west1/subnet-1.roles/compute.networkUser"]: + module.vpc-host.google_compute_subnetwork_iam_binding.authoritative["europe-west1/subnet-1.roles/compute.networkUser"]: condition: [] members: - serviceAccount:cloudsvc @@ -39,7 +39,7 @@ values: region: europe-west1 role: roles/compute.networkUser subnetwork: subnet-1 - module.vpc-host.google_compute_subnetwork_iam_binding.binding["europe-west1/subnet-1.roles/compute.securityAdmin"]: + module.vpc-host.google_compute_subnetwork_iam_binding.authoritative["europe-west1/subnet-1.roles/compute.securityAdmin"]: condition: [] members: - serviceAccount:gke diff --git a/tests/modules/net_vpc/examples/subnet-iam.yaml b/tests/modules/net_vpc/examples/subnet-iam.yaml index 2c6dcccc..68b03418 100644 --- a/tests/modules/net_vpc/examples/subnet-iam.yaml +++ b/tests/modules/net_vpc/examples/subnet-iam.yaml @@ -71,7 +71,7 @@ values: role: null secondary_ip_range: [] timeouts: null - module.vpc.google_compute_subnetwork_iam_binding.binding["europe-west1/subnet-1.roles/compute.networkUser"]: + module.vpc.google_compute_subnetwork_iam_binding.authoritative["europe-west1/subnet-1.roles/compute.networkUser"]: condition: [] members: - group:group1@example.com @@ -80,21 +80,18 @@ values: region: europe-west1 role: roles/compute.networkUser subnetwork: subnet-1 - ? module.vpc.google_compute_subnetwork_iam_member.binding["europe-west1/subnet-2.roles/compute.networkUser.group:group2@example.com"] - : condition: [] - member: group:group2@example.com + module.vpc.google_compute_subnetwork_iam_binding.bindings["europe-west1/subnet-1.roles/compute.networkUser.test_condition"]: + condition: + - description: null + expression: resource.matchTag('123456789012/env', 'prod') + title: test_condition + members: + - group:group2@example.com project: my-project region: europe-west1 role: roles/compute.networkUser - subnetwork: subnet-2 - ? module.vpc.google_compute_subnetwork_iam_member.binding["europe-west1/subnet-2.roles/compute.networkUser.user:user2@example.com"] - : condition: [] - member: user:user2@example.com - project: my-project - region: europe-west1 - role: roles/compute.networkUser - subnetwork: subnet-2 - module.vpc.google_compute_subnetwork_iam_member.members["subnet-2-am1"]: + subnetwork: subnet-1 + module.vpc.google_compute_subnetwork_iam_member.bindings["subnet-2-am1"]: condition: [] member: user:am1@example.com project: my-project @@ -106,9 +103,9 @@ counts: google_compute_network: 1 google_compute_route: 2 google_compute_subnetwork: 2 - google_compute_subnetwork_iam_binding: 1 - google_compute_subnetwork_iam_member: 3 + google_compute_subnetwork_iam_binding: 2 + google_compute_subnetwork_iam_member: 1 modules: 1 - resources: 9 + resources: 8 outputs: {} diff --git a/tests/modules/net_vpc/psa_routes_export.yaml b/tests/modules/net_vpc/psa_routes_export.yaml index c8cf631b..fd9239e8 100644 --- a/tests/modules/net_vpc/psa_routes_export.yaml +++ b/tests/modules/net_vpc/psa_routes_export.yaml @@ -46,7 +46,6 @@ counts: google_service_networking_connection: 1 outputs: - bindings: {} name: __missing__ network: __missing__ project_id: test-project diff --git a/tests/modules/net_vpc/psa_routes_import.yaml b/tests/modules/net_vpc/psa_routes_import.yaml index 771d881f..dc0e5704 100644 --- a/tests/modules/net_vpc/psa_routes_import.yaml +++ b/tests/modules/net_vpc/psa_routes_import.yaml @@ -46,7 +46,6 @@ counts: google_service_networking_connection: 1 outputs: - bindings: {} name: __missing__ network: __missing__ project_id: test-project diff --git a/tests/modules/net_vpc/psa_routes_import_export.yaml b/tests/modules/net_vpc/psa_routes_import_export.yaml index 561fa50a..23db6b26 100644 --- a/tests/modules/net_vpc/psa_routes_import_export.yaml +++ b/tests/modules/net_vpc/psa_routes_import_export.yaml @@ -46,7 +46,6 @@ counts: google_service_networking_connection: 1 outputs: - bindings: {} name: __missing__ network: __missing__ project_id: test-project diff --git a/tests/modules/net_vpc/shared_vpc.yaml b/tests/modules/net_vpc/shared_vpc.yaml index 0837dbc4..5b6ffd3e 100644 --- a/tests/modules/net_vpc/shared_vpc.yaml +++ b/tests/modules/net_vpc/shared_vpc.yaml @@ -35,7 +35,6 @@ counts: google_compute_shared_vpc_service_project: 2 outputs: - bindings: {} project_id: test-project subnet_ips: {} subnet_regions: {} diff --git a/tests/modules/organization/examples/basic.yaml b/tests/modules/organization/examples/basic.yaml index 3c8bf8a1..d1e0a118 100644 --- a/tests/modules/organization/examples/basic.yaml +++ b/tests/modules/organization/examples/basic.yaml @@ -166,17 +166,7 @@ values: - group:cloud-admins@example.org org_id: '1234567890' role: roles/resourcemanager.projectCreator - module.org.google_organization_iam_member.additive["roles/compute.admin-user:compute@example.org"]: - condition: [] - member: user:compute@example.org - org_id: '1234567890' - role: roles/compute.admin - module.org.google_organization_iam_member.additive["roles/container.viewer-user:compute@example.org"]: - condition: [] - member: user:compute@example.org - org_id: '1234567890' - role: roles/container.viewer - module.org.google_organization_iam_member.members["am1-storage-admin"]: + module.org.google_organization_iam_member.bindings["am1-storage-admin"]: condition: [] member: user:am1@example.org org_id: '1234567890' @@ -200,10 +190,10 @@ values: counts: google_org_policy_policy: 8 google_organization_iam_binding: 3 - google_organization_iam_member: 3 + google_organization_iam_member: 1 google_tags_tag_key: 1 google_tags_tag_value: 2 modules: 1 - resources: 17 + resources: 15 outputs: {} diff --git a/tests/modules/organization/examples/iam-policy.yaml b/tests/modules/organization/examples/iam-policy.yaml deleted file mode 100644 index d853ca64..00000000 --- a/tests/modules/organization/examples/iam-policy.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.org.google_organization_iam_policy.authoritative[0]: - org_id: '1122334455' - policy_data: '{"auditConfigs":[{"auditLogConfigs":[{"exemptedMembers":["group:organization-admins@example.org"],"logType":"ADMIN_READ"}],"service":"allServices"},{"auditLogConfigs":[{"logType":"DATA_WRITE"},{"logType":"DATA_READ"}],"service":"storage.googleapis.com"}],"bindings":[{"members":["group:org-admins@example.com"],"role":"roles/owner"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.folderAdmin"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.organizationAdmin"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.projectCreator"}]}' - -counts: - google_organization_iam_policy: 1 - modules: 1 - resources: 1 diff --git a/tests/modules/organization/test_plan_org_policies_modules.py b/tests/modules/organization/test_plan_org_policies_modules.py index 3126872e..4d92b46f 100644 --- a/tests/modules/organization/test_plan_org_policies_modules.py +++ b/tests/modules/organization/test_plan_org_policies_modules.py @@ -57,9 +57,9 @@ def test_policy_implementation(): '@@ -116,0 +117,9 @@\n', '+ depends_on = [\n', '+ google_organization_iam_binding.authoritative,\n', + '+ google_organization_iam_binding.bindings,\n', + '+ google_organization_iam_member.bindings,\n', '+ google_organization_iam_custom_role.roles,\n', - '+ google_organization_iam_member.additive,\n', - '+ google_organization_iam_policy.authoritative,\n', '+ google_org_policy_custom_constraint.constraint,\n', '+ google_tags_tag_key.default,\n', '+ google_tags_tag_value.default,\n', diff --git a/tests/modules/project/examples/iam-additive-members.yaml b/tests/modules/project/examples/iam-additive-members.yaml deleted file mode 100644 index 6a517a4a..00000000 --- a/tests/modules/project/examples/iam-additive-members.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.project.google_project.project[0]: - project_id: project-example - module.project.google_project_iam_member.additive["roles/editor-user:two@example.org"]: - condition: [] - project: project-example - role: roles/editor - module.project.google_project_iam_member.additive["roles/owner-user:one@example.org"]: - condition: [] - project: project-example - role: roles/owner - module.project.google_project_iam_member.additive["roles/owner-user:two@example.org"]: - condition: [] - project: project-example - role: roles/owner - -counts: - google_project: 1 - google_project_iam_member: 3 diff --git a/tests/modules/project/examples/iam-additive.yaml b/tests/modules/project/examples/iam-additive.yaml deleted file mode 100644 index 5bab8223..00000000 --- a/tests/modules/project/examples/iam-additive.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.project.google_project.project[0]: {} - module.project.google_project_iam_member.additive["roles/owner-group:three@example.org"]: - condition: [] - project: project-example - role: roles/owner - module.project.google_project_iam_member.additive["roles/storage.objectAdmin-group:two@example.org"]: - condition: [] - project: project-example - role: roles/storage.objectAdmin - module.project.google_project_iam_member.additive["roles/viewer-group:one@example.org"]: - condition: [] - project: project-example - role: roles/viewer - module.project.google_project_iam_member.additive["roles/viewer-group:two@xample.org"]: - condition: [] - project: project-example - role: roles/viewer - -counts: - google_project: 1 - google_project_iam_member: 4 diff --git a/tests/modules/project/examples/iam-authoritative.yaml b/tests/modules/project/examples/iam-authoritative.yaml index f190a429..d6e33ab2 100644 --- a/tests/modules/project/examples/iam-authoritative.yaml +++ b/tests/modules/project/examples/iam-authoritative.yaml @@ -13,7 +13,16 @@ # limitations under the License. values: - module.project.google_project.project[0]: {} + module.project.google_project.project[0]: + auto_create_network: false + billing_account: 123456-123456-123456 + folder_id: '1234567890' + labels: null + name: foo-project-example + org_id: null + project_id: foo-project-example + skip_delete: false + timeouts: null module.project.google_project_iam_binding.authoritative["roles/container.hostServiceAgentUser"]: condition: [] members: @@ -37,3 +46,8 @@ counts: google_project: 1 google_project_iam_binding: 1 google_project_service: 2 + modules: 1 + resources: 4 + +outputs: {} + diff --git a/tests/modules/project/examples/iam-bindings-additive.yaml b/tests/modules/project/examples/iam-bindings-additive.yaml new file mode 100644 index 00000000..390781f9 --- /dev/null +++ b/tests/modules/project/examples/iam-bindings-additive.yaml @@ -0,0 +1,46 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.project.google_project.project[0]: + auto_create_network: false + billing_account: null + folder_id: null + labels: null + name: project-1 + org_id: null + project_id: project-1 + skip_delete: false + timeouts: null + module.project.google_project_iam_member.bindings["group-owner"]: + condition: [] + member: group:p1-owners@example.org + project: project-1 + role: roles/owner + module.project.google_project_service.project_services["compute.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: project-1 + service: compute.googleapis.com + timeouts: null + +counts: + google_project: 1 + google_project_iam_member: 1 + google_project_service: 1 + modules: 1 + resources: 3 + +outputs: {} + diff --git a/tests/modules/project/examples/iam-members.yaml b/tests/modules/project/examples/iam-bindings.yaml similarity index 54% rename from tests/modules/project/examples/iam-members.yaml rename to tests/modules/project/examples/iam-bindings.yaml index 41f48a56..f1f09e36 100644 --- a/tests/modules/project/examples/iam-members.yaml +++ b/tests/modules/project/examples/iam-bindings.yaml @@ -15,44 +15,43 @@ values: module.project.google_project.project[0]: auto_create_network: false - billing_account: null - folder_id: null + billing_account: 123456-123456-123456 + folder_id: '1234567890' labels: null - name: project-example + name: foo-project-example org_id: null - project_id: project-example + project_id: foo-project-example skip_delete: false timeouts: null - module.project.google_project_iam_member.members["one-delegated-grant"]: + module.project.google_project_iam_binding.bindings["roles/resourcemanager.projectIamAdmin"]: condition: - description: null expression: "api.getAttribute(\n 'iam.googleapis.com/modifiedGrantsByRole',\ \ []\n).hasOnly([\n 'roles/compute.networkAdmin'\n])\n" title: delegated_network_user_one - member: user:one@example.org - project: project-example + members: + - group:test-admins@example.org + project: foo-project-example role: roles/resourcemanager.projectIamAdmin - module.project.google_project_iam_member.members["one-owner"]: - condition: [] - member: user:one@example.org - project: project-example - role: roles/owner - module.project.google_project_iam_member.members["two-compute-admin"]: - condition: [] - member: user:two@example.org - project: project-example - role: roles/compute.admin - module.project.google_project_iam_member.members["two-viewer"]: - condition: [] - member: user:two@example.org - project: project-example - role: roles/viewer + module.project.google_project_service.project_services["container.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: foo-project-example + service: container.googleapis.com + timeouts: null + module.project.google_project_service.project_services["stackdriver.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: foo-project-example + service: stackdriver.googleapis.com + timeouts: null counts: google_project: 1 - google_project_iam_member: 4 + google_project_iam_binding: 1 + google_project_service: 2 modules: 1 - resources: 5 + resources: 4 outputs: {} diff --git a/tests/modules/project/examples/iam-policy.yaml b/tests/modules/project/examples/iam-policy.yaml deleted file mode 100644 index 5aebc110..00000000 --- a/tests/modules/project/examples/iam-policy.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.project.google_project.project[0]: - auto_create_network: false - billing_account: 123456-123456-123456 - folder_id: '1234567890' - labels: null - name: my-project - org_id: null - project_id: my-project - skip_delete: false - timeouts: null - module.project.google_project_iam_policy.authoritative[0]: - policy_data: '{"auditConfigs":[{"auditLogConfigs":[{"exemptedMembers":["group:organization-admins@example.org"],"logType":"ADMIN_READ"}],"service":"allServices"},{"auditLogConfigs":[{"logType":"DATA_WRITE"},{"logType":"DATA_READ"}],"service":"storage.googleapis.com"}],"bindings":[{"members":["group:org-admins@example.com"],"role":"roles/owner"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.folderAdmin"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.organizationAdmin"},{"members":["group:org-admins@example.com"],"role":"roles/resourcemanager.projectCreator"}]}' - project: my-project - -counts: - google_project: 1 - google_project_iam_policy: 1 - modules: 1 - resources: 2 diff --git a/tests/modules/source_repository/examples/simple.yaml b/tests/modules/source_repository/examples/simple.yaml index 249b6af1..268d143a 100644 --- a/tests/modules/source_repository/examples/simple.yaml +++ b/tests/modules/source_repository/examples/simple.yaml @@ -25,7 +25,7 @@ values: project: my-project repository: my-repo role: roles/source.reader - module.repo.google_sourcerepo_repository_iam_member.members["am1-reader"]: + module.repo.google_sourcerepo_repository_iam_member.bindings["am1-reader"]: condition: [] member: user:am1@example.com project: my-project @@ -40,3 +40,4 @@ counts: resources: 3 outputs: {} + From 1c1446f4c2b6c70dbb5e8361ab2cbf7609edc979 Mon Sep 17 00:00:00 2001 From: Ludo Date: Sun, 20 Aug 2023 18:37:31 +0200 Subject: [PATCH 41/46] update changelog --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8e5a07f..8d81ef14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,29 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### BLUEPRINTS + +- [[#1595](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1595)] **incompatible change:** IAM interface refactor ([ludoo](https://github.com/ludoo)) +- [[#1601](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1601)] [Data Platform] Update README.md ([lcaggio](https://github.com/lcaggio)) + +### DOCUMENTATION + +- [[#1595](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1595)] **incompatible change:** IAM interface refactor ([ludoo](https://github.com/ludoo)) + ### FAST +- [[#1595](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1595)] **incompatible change:** IAM interface refactor ([ludoo](https://github.com/ludoo)) +- [[#1597](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1597)] fix null object exception in bootstrap output when using cloudsource ([sm3142](https://github.com/sm3142)) - [[#1593](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1593)] Fix FAST CI/CD for Gitlab ([ludoo](https://github.com/ludoo)) - [[#1583](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1583)] Fix module path for teams cicd ([ludoo](https://github.com/ludoo)) ### MODULES +- [[#1595](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1595)] **incompatible change:** IAM interface refactor ([ludoo](https://github.com/ludoo)) +- [[#1600](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1600)] fix(cloud-run): move cpu boost annotation to revision ([LiuVII](https://github.com/LiuVII)) +- [[#1599](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1599)] Fixing some typos ([bluPhy](https://github.com/bluPhy)) +- [[#1598](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1598)] feat(cloud-run): add startup cpu boost option ([JSchwerberg](https://github.com/JSchwerberg)) +- [[#1594](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1594)] Add support for conditions to `iam_members` module variables ([ludoo](https://github.com/ludoo)) - [[#1591](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1591)] feat: 🎸 (modules/cloudsql-instance):add project_id for ssl cert ([erabusi](https://github.com/erabusi)) - [[#1589](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1589)] Add new `iam_members` variable to IAM additive module interfaces ([ludoo](https://github.com/ludoo)) - [[#1588](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1588)] feat: 🎸 (modules/cloudsql-instance): enable require_ssl cert support ([erabusi](https://github.com/erabusi)) @@ -22,6 +38,7 @@ All notable changes to this project will be documented in this file. ### TOOLS +- [[#1595](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1595)] **incompatible change:** IAM interface refactor ([ludoo](https://github.com/ludoo)) - [[#1585](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1585)] Print inventory path when a test fails ([juliocc](https://github.com/juliocc)) ## [25.0.0] - 2023-08-09 From e43be5b387b4c7eb44b7a7fdbc98a7991d173b33 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 21 Aug 2023 07:01:06 +0200 Subject: [PATCH 42/46] Update README.md --- modules/__docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/__docs/README.md b/modules/__docs/README.md index d6c722c2..da5c9181 100644 --- a/modules/__docs/README.md +++ b/modules/__docs/README.md @@ -1,3 +1,3 @@ -# FAST architectural documents +# Fabric modules architectural documents This folder contains assorted bits of documentation used to log current architectural choices, or past decisions. Format is inspired by [Michael Nygard's decision record template](https://github.com/joelparkerhenderson/architecture-decision-record/blob/main/templates/decision-record-template-by-michael-nygard/index.md). From 63b0480499be3265dc99f8720dc85ab655eb01dd Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 21 Aug 2023 07:09:54 +0200 Subject: [PATCH 43/46] Update 20230816-iam-refactor.md --- modules/__docs/20230816-iam-refactor.md | 42 +++++++------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/modules/__docs/20230816-iam-refactor.md b/modules/__docs/20230816-iam-refactor.md index a252d27a..06acc1ee 100644 --- a/modules/__docs/20230816-iam-refactor.md +++ b/modules/__docs/20230816-iam-refactor.md @@ -5,7 +5,7 @@ ## Status -Discussed. +Implemented in #1595. ## Context @@ -115,34 +115,16 @@ The proposal above summarizes the state of discussions between the authors, and ## Consequences -A few data blueprints that leverage `iam_additive` will need to be refactored to use the new variable, using one of the following patterns: +### FAST -```hcl -locals { - network_sa_roles = [ - "roles/compute.orgFirewallPolicyAdmin", - "roles/compute.xpnAdmin" - ] -} +IAM implementation in the bootstrap stage and matching multitenant bootstrap has radically changed, with the addition of a new [`organization-iam.tf`](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/blob/master/fast/stages/0-bootstrap/organization-iam.tf) file which contains IAM binding definitions in an abstracted format, that is then converted to the specific formats required by the `iam`, `iam_bindings` and `iam_bindings_additive` variables. -module "organization" { - source = "../../../modules/organization" - organization_id = "organizations/${var.organization.id}" - iam_bindings_additive = merge( - # IAM bindings via locals pattern - { - for r in local.network_sa_roles : "network_sa-${r}" : { - member = module.branch-network-sa.iam_email - role = r - } - }, - # IAM bindings via explicit reference pattern - { - security_sa = { - member = module.branch-security-sa.iam_email - role = "roles/accesscontextmanager.policyAdmin" - } - } - ) -} -``` +This brings several advantages over the previous handling of IAM: + +- authoritative and additive bindings are now grouped by principal in an easy to read and change format that serves as its own documentation +- support for IAM conditions has removed the need for standalone resources and made the intent behind those more explicit +- some subtle bugs on the intersection of user-specified bindings and internally-specified ones have been addressed + +### Blueprints + +A few data blueprints that leverage `iam_additive` have been refactored to use the new variable. This is most notable in data blueprints, where extra files have been added to the more complex examples like data foundations, to abstract IAM bindings in a way similar to what is described above for FAST. From e7eeed12f8bdc7e970843d6cc9a3a96f3dd54128 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 21 Aug 2023 07:10:29 +0200 Subject: [PATCH 44/46] Update 20230816-iam-refactor.md --- modules/__docs/20230816-iam-refactor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/__docs/20230816-iam-refactor.md b/modules/__docs/20230816-iam-refactor.md index 06acc1ee..99af5bff 100644 --- a/modules/__docs/20230816-iam-refactor.md +++ b/modules/__docs/20230816-iam-refactor.md @@ -5,7 +5,7 @@ ## Status -Implemented in #1595. +Implemented in [#1595](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1595). ## Context From 0d17af7967b0a99463c709ac6cd00be68b4310f7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 21 Aug 2023 07:11:24 +0200 Subject: [PATCH 45/46] Update 20230816-iam-refactor.md --- modules/__docs/20230816-iam-refactor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/__docs/20230816-iam-refactor.md b/modules/__docs/20230816-iam-refactor.md index 99af5bff..438252ac 100644 --- a/modules/__docs/20230816-iam-refactor.md +++ b/modules/__docs/20230816-iam-refactor.md @@ -9,7 +9,7 @@ Implemented in [#1595](https://github.com/GoogleCloudPlatform/cloud-foundation-f ## Context -Our modules IAM interface has evolved organically to progressively support more functionality, resulting in a large variable surface, lack of support for some key features like conditions, and some fragility for specific use cases. +The IAM interface in our modules has evolved organically to progressively support more functionality, resulting in a large variable surface, lack of support for some key features like conditions, and some fragility for specific use cases. We currently support, with uneven coverage across modules: From 91f71fef6816cf3191255946c3df262a08f86268 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 21 Aug 2023 18:37:54 +0200 Subject: [PATCH 46/46] trap requests timeout error (#1607) --- blueprints/cloud-operations/quota-monitoring/src/main.py | 4 +++- .../cloud-operations/quota-monitoring/src/requirements.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index ecdacbe2..5a845364 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -29,6 +29,7 @@ import warnings import click import google.auth +import requests.exceptions from google.auth.transport.requests import AuthorizedSession @@ -123,7 +124,8 @@ def fetch(request, delete=False): else: response = HTTP.post(request.url, headers=request.headers, data=json.dumps(request.data)) - except google.auth.exceptions.RefreshError as e: + except (google.auth.exceptions.RefreshError, + requests.exceptions.ReadTimeout) as e: raise SystemExit(e.args[0]) try: rdata = json.loads(response.content) diff --git a/blueprints/cloud-operations/quota-monitoring/src/requirements.txt b/blueprints/cloud-operations/quota-monitoring/src/requirements.txt index f488fac7..408307eb 100644 --- a/blueprints/cloud-operations/quota-monitoring/src/requirements.txt +++ b/blueprints/cloud-operations/quota-monitoring/src/requirements.txt @@ -2,3 +2,4 @@ click functions-framework google-api-core google-cloud-monitoring +requests \ No newline at end of file