From 40cb46e1cc59c36cac9dd3198c841f32cee11733 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 9 Feb 2022 11:06:51 +0100 Subject: [PATCH 1/6] Add support for Shared VPC service IAM to project module (#525) * project module changes * fix examples * add comments in module code * re-enable nullable on svpc variables * project factory * Tests still failing (#526) * fix pf * tfdoc * pf test boilerplate Co-authored-by: Simone Ruffilli --- examples/factories/project-factory/README.md | 6 +- examples/factories/project-factory/main.tf | 149 +++++++++++------- examples/factories/project-factory/outputs.tf | 6 +- .../factories/project-factory/variables.tf | 6 +- examples/networking/filtering-proxy/main.tf | 4 +- examples/networking/shared-vpc-gke/main.tf | 12 +- fast/assets/schemas/project.schema.yaml | 2 +- .../prod/data/projects/project.yaml | 4 +- modules/project/README.md | 55 +++++-- modules/project/outputs.tf | 3 + modules/project/service-accounts.tf | 11 +- modules/project/shared-vpc.tf | 50 +++++- modules/project/variables.tf | 17 +- .../factories/project_factory/__init__.py | 13 ++ .../project_factory/fixture/defaults.yaml | 24 +++ .../factories/project_factory/fixture/main.tf | 51 ++++++ .../fixture/projects/project.yaml | 100 ++++++++++++ .../project_factory/fixture/variables.tf | 52 ++++++ .../factories/project_factory/test_plan.py | 18 +++ .../networking/filtering_proxy/test_plan.py | 4 +- .../networking/shared_vpc_gke/test_plan.py | 2 +- 21 files changed, 479 insertions(+), 110 deletions(-) create mode 100644 tests/examples/factories/project_factory/__init__.py create mode 100644 tests/examples/factories/project_factory/fixture/defaults.yaml create mode 100644 tests/examples/factories/project_factory/fixture/main.tf create mode 100644 tests/examples/factories/project_factory/fixture/projects/project.yaml create mode 100644 tests/examples/factories/project_factory/fixture/variables.tf create mode 100644 tests/examples/factories/project_factory/test_plan.py diff --git a/examples/factories/project-factory/README.md b/examples/factories/project-factory/README.md index f815f9d5..0b8b5183 100644 --- a/examples/factories/project-factory/README.md +++ b/examples/factories/project-factory/README.md @@ -231,15 +231,15 @@ vpc: | [org_policies](variables.tf#L98) | Org-policy overrides at project level. | object({…}) | | null | | [prefix](variables.tf#L112) | Prefix used for the project id. | string | | null | | [service_accounts](variables.tf#L123) | Service accounts to be created, and roles to assign them. | map(list(string)) | | {} | +| [service_identities_iam](variables.tf#L136) | Custom IAM settings for service identities in service => [role] format. | map(list(string)) | | {} | | [services](variables.tf#L129) | Services to be enabled for the project. | list(string) | | [] | -| [services_iam](variables.tf#L135) | Custom IAM settings for robot ServiceAccounts in service => [role] format. | map(list(string)) | | {} | -| [vpc](variables.tf#L141) | VPC configuration for the project. | object({…}) | | null | +| [vpc](variables.tf#L143) | VPC configuration for the project. | object({…}) | | null | ## Outputs | name | description | sensitive | |---|---|:---:| | [project](outputs.tf#L19) | The project resource as return by the `project` module | | -| [project_id](outputs.tf#L30) | Project ID. | | +| [project_id](outputs.tf#L29) | Project ID. | | diff --git a/examples/factories/project-factory/main.tf b/examples/factories/project-factory/main.tf index df449edb..5f8356fc 100644 --- a/examples/factories/project-factory/main.tf +++ b/examples/factories/project-factory/main.tf @@ -15,24 +15,26 @@ */ locals { - _gke_iam_hsau = try(var.vpc.gke_setup.enable_host_service_agent, false) ? { - "roles/container.hostServiceAgentUser" = "serviceAccount:${module.project.service_accounts.robots.container-engine}" - } : {} - _gke_iam_securityadmin = try(var.vpc.gke_setup.enable_security_admin, false) ? { - "roles/compute.securityAdmin" = "serviceAccount:${module.project.service_accounts.robots.container-engine}" - } : {} + # internal structures for group IAM bindings _group_iam = { - for r in local._group_iam_roles : r => [ - for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + for r in local._group_iam_bindings : r => [ + for k, v in var.group_iam : + "group:${k}" if try(index(v, r), null) != null ] } - _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam_bindings = distinct(flatten(values(var.group_iam))) + # internal structures for project service accounts IAM bindings _service_accounts_iam = { - for r in local._service_accounts_iam_roles : r => [ - for k, v in var.service_accounts : "serviceAccount:${k}@${var.project_id}.iam.gserviceaccount.com" if try(index(v, r), null) != null + for r in local._service_accounts_iam_bindings : r => [ + for k, v in var.service_accounts : + "serviceAccount:${k}@${var.project_id}.iam.gserviceaccount.com" + if try(index(v, r), null) != null ] } - _service_accounts_iam_roles = distinct(flatten(values(var.service_accounts))) + _service_accounts_iam_bindings = distinct(flatten( + values(var.service_accounts) + )) + # internal structures for project services _services = concat([ "billingbudgets.googleapis.com", "essentialcontacts.googleapis.com" @@ -41,46 +43,81 @@ locals { try(var.vpc.gke_setup, null) != null ? ["container.googleapis.com"] : [], var.vpc != null ? ["compute.googleapis.com"] : [], ) - _services_iam_roles = distinct(flatten(values(var.services_iam))) - _services_iam = { - for r in local._services_iam_roles : r => [ - for k, v in var.services_iam : "serviceAccount:${module.project.service_accounts.robots[k]}" if try(index(v, r), null) != null + # internal structures for service identity IAM bindings + _service_identities_roles = distinct(flatten(values(var.service_identities_iam))) + _service_identities_iam = { + for role in local._service_identities_roles : role => [ + for service, roles in var.service_identities_iam : + "serviceAccount:${module.project.service_accounts.robots[service]}" + if contains(roles, role) ] } - billing_account_id = coalesce(var.billing_account_id, try(var.defaults.billing_account_id, "")) - billing_alert = var.billing_alert == null ? try(var.defaults.billing_alert, null) : var.billing_alert - essential_contacts = concat(try(var.defaults.essential_contacts, []), var.essential_contacts) - host_project_bindings = merge( - local._gke_iam_hsau, - local._gke_iam_securityadmin + # internal structure for Shared VPC service project IAM bindings + _vpc_subnet_bindings = ( + local.vpc.subnets_iam == null || local.vpc.host_project == null + ? [] + : flatten([ + for subnet, members in local.vpc.subnets_iam : [ + for member in members : { + region = split("/", subnet)[0] + subnet = split("/", subnet)[1] + member = member + } + ] + ]) ) + # structures for billing id + billing_account_id = coalesce( + var.billing_account_id, try(var.defaults.billing_account_id, "") + ) + billing_alert = ( + var.billing_alert == null + ? try(var.defaults.billing_alert, null) + : var.billing_alert + ) + # structure for essential contacts + essential_contacts = concat( + try(var.defaults.essential_contacts, []), var.essential_contacts + ) + # structure that combines all authoritative IAM bindings iam = { for role in distinct(concat( keys(var.iam), keys(local._group_iam), keys(local._service_accounts_iam), - keys(local._services_iam), + keys(local._service_identities_iam), )) : role => concat( try(var.iam[role], []), try(local._group_iam[role], []), try(local._service_accounts_iam[role], []), - try(local._services_iam[role], []), + try(local._service_identities_iam[role], []), ) } - labels = merge(coalesce(var.labels, {}), coalesce(try(var.defaults.labels, {}), {})) - network_user_service_accounts = concat( - contains(local.services, "compute.googleapis.com") ? [ - "serviceAccount:${module.project.service_accounts.robots.compute}" - ] : [], - contains(local.services, "container.googleapis.com") ? [ - "serviceAccount:${module.project.service_accounts.robots.container-engine}", - "serviceAccount:${module.project.service_accounts.cloud_services}" - ] : [], - []) - services = distinct(concat(var.services, local._services)) - vpc_host_project = try(var.vpc.host_project, var.defaults.vpc_host_project) - vpc_setup = var.vpc != null + # merge labels with defaults + labels = merge( + coalesce(var.labels, {}), coalesce(try(var.defaults.labels, {}), {}) + ) + # deduplicate services + services = distinct(concat(var.services, local._services)) + # structures for Shared VPC resources in host project + vpc = coalesce(var.vpc, { + host_project = null, gke_setup = null, subnets_iam = null + }) + vpc_cloudservices = ( + local.vpc_gke_service_agent || + contains(var.services, "compute.googleapis.com") + ) + vpc_gke_security_admin = coalesce( + try(local.vpc.gke_setup.enable_security_admin, null), false + ) + vpc_gke_service_agent = coalesce( + try(local.vpc.gke_setup.enable_host_service_agent, null), false + ) + vpc_subnet_bindings = { + for binding in local._vpc_subnet_bindings : + "${binding.subnet}:${binding.member}" => binding + } } module "billing-alert" { @@ -122,9 +159,21 @@ module "project" { policy_list = try(var.org_policies.policy_list, {}) service_encryption_key_ids = var.kms_service_agents services = local.services - shared_vpc_service_config = { - attach = local.vpc_setup - host_project = local.vpc_host_project + shared_vpc_service_config = var.vpc == null ? null : { + host_project = local.vpc.host_project + # these are non-authoritative + service_identity_iam = { + "roles/compute.networkUser" = compact([ + local.vpc_gke_service_agent ? "container-engine" : null, + local.vpc_cloudservices ? "cloudservices" : null + ]) + "roles/compute.securityAdmin" = compact([ + local.vpc_gke_security_admin ? "container-engine" : null, + ]) + "roles/container.hostServiceAgentUser" = compact([ + local.vpc_gke_service_agent ? "container-engine" : null + ]) + } } } @@ -135,19 +184,11 @@ module "service-accounts" { project_id = module.project.project_id } -# TODO(jccb): we should probably change this to non-authoritative bindings -resource "google_compute_subnetwork_iam_binding" "binding" { - for_each = local.vpc_setup ? coalesce(var.vpc.subnets_iam, {}) : {} - project = local.vpc_host_project - subnetwork = "projects/${local.vpc_host_project}/regions/${split("/", each.key)[0]}/subnetworks/${split("/", each.key)[1]}" - region = split("/", each.key)[0] +resource "google_compute_subnetwork_iam_member" "default" { + for_each = local.vpc_subnet_bindings + project = local.vpc.host_project + subnetwork = "projects/${local.vpc.host_project}/regions/${each.value.region}/subnetworks/${each.value.subnet}" + region = each.value.region role = "roles/compute.networkUser" - members = concat(each.value, local.network_user_service_accounts) -} - -resource "google_project_iam_member" "host_project_bindings" { - for_each = local.host_project_bindings - project = local.vpc_host_project - role = each.key - member = each.value + member = each.value.member } diff --git a/examples/factories/project-factory/outputs.tf b/examples/factories/project-factory/outputs.tf index 7504eda5..a60ad457 100644 --- a/examples/factories/project-factory/outputs.tf +++ b/examples/factories/project-factory/outputs.tf @@ -21,8 +21,7 @@ output "project" { value = module.project depends_on = [ - google_compute_subnetwork_iam_binding.binding, - google_project_iam_member.host_project_bindings, + google_compute_subnetwork_iam_member.default, module.dns ] } @@ -31,8 +30,7 @@ output "project_id" { description = "Project ID." value = module.project.project_id depends_on = [ - google_compute_subnetwork_iam_binding.binding, - google_project_iam_member.host_project_bindings, + google_compute_subnetwork_iam_member.default, module.dns ] } diff --git a/examples/factories/project-factory/variables.tf b/examples/factories/project-factory/variables.tf index 777dce71..7f4a20f7 100644 --- a/examples/factories/project-factory/variables.tf +++ b/examples/factories/project-factory/variables.tf @@ -130,12 +130,14 @@ variable "services" { description = "Services to be enabled for the project." type = list(string) default = [] + nullable = false } -variable "services_iam" { - description = "Custom IAM settings for robot ServiceAccounts in service => [role] format." +variable "service_identities_iam" { + description = "Custom IAM settings for service identities in service => [role] format." type = map(list(string)) default = {} + nullable = false } variable "vpc" { diff --git a/examples/networking/filtering-proxy/main.tf b/examples/networking/filtering-proxy/main.tf index cae1eb84..440a4b59 100644 --- a/examples/networking/filtering-proxy/main.tf +++ b/examples/networking/filtering-proxy/main.tf @@ -252,8 +252,10 @@ module "project-app" { prefix = var.prefix services = ["compute.googleapis.com"] shared_vpc_service_config = { - attach = true host_project = module.project-host.project_id + service_identity_iam = { + "roles/compute.networkUser" = ["cloudservices"] + } } } diff --git a/examples/networking/shared-vpc-gke/main.tf b/examples/networking/shared-vpc-gke/main.tf index b6af786b..9ee388ba 100644 --- a/examples/networking/shared-vpc-gke/main.tf +++ b/examples/networking/shared-vpc-gke/main.tf @@ -31,9 +31,6 @@ module "project-host" { service_projects = [] # defined later } iam = { - "roles/container.hostServiceAgentUser" = [ - "serviceAccount:${module.project-svc-gke.service_accounts.robots.container-engine}" - ] "roles/owner" = var.owners_host } } @@ -48,8 +45,10 @@ module "project-svc-gce" { oslogin = true oslogin_admins = var.owners_gce shared_vpc_service_config = { - attach = true host_project = module.project-host.project_id + service_identity_iam = { + "roles/compute.networkUser" = ["cloudservices"] + } } iam = { "roles/owner" = var.owners_gce @@ -67,8 +66,11 @@ module "project-svc-gke" { name = "gke" services = var.project_services shared_vpc_service_config = { - attach = true host_project = module.project-host.project_id + service_identity_iam = { + "roles/container.hostServiceAgentUser" = ["container-engine"] + "roles/compute.networkUser" = ["container-engine"] + } } iam = merge( { diff --git a/fast/assets/schemas/project.schema.yaml b/fast/assets/schemas/project.schema.yaml index f7f89730..49e4bf89 100644 --- a/fast/assets/schemas/project.schema.yaml +++ b/fast/assets/schemas/project.schema.yaml @@ -25,7 +25,7 @@ org_policies: include('org_policies', required=False) secrets: map(list(str()), key=str(), required=False) service_accounts: map(list(str()), required=False) services: list(str(matches='^[a-z-]*\.googleapis\.com$'), required=False) -services_iam: map(list(str()), key=str(), required=False) +service_identities_iam: map(list(str()), key=str(), required=False) vpc: include('vpc', required=False) --- billing_alert: diff --git a/fast/stages/03-project-factory/prod/data/projects/project.yaml b/fast/stages/03-project-factory/prod/data/projects/project.yaml index 244f6955..7ad16016 100644 --- a/fast/stages/03-project-factory/prod/data/projects/project.yaml +++ b/fast/stages/03-project-factory/prod/data/projects/project.yaml @@ -72,8 +72,8 @@ services: - stackdriver.googleapis.com - compute.googleapis.com -# [opt] Roles to assign to the robots service accounts in robot => [roles] format -services_iam: +# [opt] Roles to assign to the service identities in service => [roles] format +service_identities_iam: compute: - roles/storage.objectViewer diff --git a/modules/project/README.md b/modules/project/README.md index ae91cf2b..864b6c26 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -36,14 +36,46 @@ module "project" { name = "project-example" iam_additive = { - "roles/viewer" = ["group:one@example.org", "group:two@xample.org"], - "roles/storage.objectAdmin" = ["group:two@example.org"], - "roles/owner" = ["group:three@example.org"], + "roles/viewer" = [ + "group:one@example.org", "group:two@xample.org" + ], + "roles/storage.objectAdmin" = [ + "group:two@example.org" + ], + "roles/owner" = [ + "group:three@example.org" + ], } } # tftest modules=1 resources=5 ``` +### Shared VPC service + +```hcl +module "project" { + source = "./modules/project" + name = "project-example" + + shared_vpc_service_config = { + attach = true + host_project = "my-host-project" + service_identity_iam = { + "roles/compute.networkUser" = [ + "cloudservices", "container-engine" + ] + "roles/vpcaccess.user" = [ + "cloudrun" + ] + "roles/container.hostServiceAgentUser" = [ + "container-engine" + ] + } + } +} +# tftest modules=1 resources=6 +``` + ### Organization policies ```hcl @@ -74,6 +106,7 @@ module "project" { ``` ## Logging Sinks + ```hcl module "gcs" { source = "./modules/gcs" @@ -187,7 +220,7 @@ module "project" { | [organization-policies.tf](./organization-policies.tf) | Project-level organization policies. | google_project_organization_policy | | [outputs.tf](./outputs.tf) | Module outputs. | | | [service-accounts.tf](./service-accounts.tf) | Service identities and supporting resources. | google_kms_crypto_key_iam_member · google_project_service_identity | -| [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | google_compute_shared_vpc_host_project · google_compute_shared_vpc_service_project | +| [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | google_compute_shared_vpc_host_project · google_compute_shared_vpc_service_project · google_project_iam_member | | [variables.tf](./variables.tf) | Module variables. | | | [versions.tf](./versions.tf) | Version pins. | | | [vpc-sc.tf](./vpc-sc.tf) | VPC-SC project-level perimeter configuration. | google_access_context_manager_service_perimeter_resource | @@ -224,9 +257,9 @@ module "project" { | [service_perimeter_bridges](variables.tf#L211) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | | [service_perimeter_standard](variables.tf#L218) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | | [services](variables.tf#L224) | Service APIs to enable. | list(string) | | [] | -| [shared_vpc_host_config](variables.tf#L230) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | {…} | -| [shared_vpc_service_config](variables.tf#L243) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | -| [skip_delete](variables.tf#L256) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | +| [shared_vpc_host_config](variables.tf#L230) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | +| [shared_vpc_service_config](variables.tf#L239) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | null | +| [skip_delete](variables.tf#L249) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | ## Outputs @@ -234,9 +267,9 @@ module "project" { |---|---|:---:| | [custom_roles](outputs.tf#L17) | Ids of the created custom roles. | | | [name](outputs.tf#L25) | Project name. | | -| [number](outputs.tf#L37) | Project number. | | -| [project_id](outputs.tf#L49) | Project id. | | -| [service_accounts](outputs.tf#L63) | Product robot service accounts in project. | | -| [sink_writer_identities](outputs.tf#L79) | Writer identities created for each sink. | | +| [number](outputs.tf#L38) | Project number. | | +| [project_id](outputs.tf#L51) | Project id. | | +| [service_accounts](outputs.tf#L66) | Product robot service accounts in project. | | +| [sink_writer_identities](outputs.tf#L82) | Writer identities created for each sink. | | diff --git a/modules/project/outputs.tf b/modules/project/outputs.tf index e523b096..10d0e558 100644 --- a/modules/project/outputs.tf +++ b/modules/project/outputs.tf @@ -30,6 +30,7 @@ output "name" { google_project_organization_policy.list, google_project_service.project_services, google_compute_shared_vpc_service_project.service_projects, + google_project_iam_member.shared_vpc_host_robots, google_kms_crypto_key_iam_member.service_identity_cmek ] } @@ -42,6 +43,7 @@ output "number" { google_project_organization_policy.list, google_project_service.project_services, google_compute_shared_vpc_service_project.service_projects, + google_project_iam_member.shared_vpc_host_robots, google_kms_crypto_key_iam_member.service_identity_cmek ] } @@ -56,6 +58,7 @@ output "project_id" { google_project_organization_policy.list, google_project_service.project_services, google_compute_shared_vpc_service_project.service_projects, + google_project_iam_member.shared_vpc_host_robots, google_kms_crypto_key_iam_member.service_identity_cmek ] } diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf index 7317e0b0..34235245 100644 --- a/modules/project/service-accounts.tf +++ b/modules/project/service-accounts.tf @@ -29,6 +29,8 @@ locals { bq = "bq-%s@bigquery-encryption" cloudasset = "service-%s@gcp-sa-cloudasset" cloudbuild = "service-%s@gcp-sa-cloudbuild" + cloudfunctions = "service-%s@gcf-admin-robot" + cloudrun = "service-%s@serverless-robot-prod" composer = "service-%s@cloudcomposer-accounts" compute = "service-%s@compute-system" container-engine = "service-%s@container-engine-robot" @@ -36,10 +38,11 @@ locals { dataflow = "service-%s@dataflow-service-producer-prod" dataproc = "service-%s@dataproc-accounts" gae-flex = "service-%s@gae-api-prod" - gcf = "service-%s@gcf-admin-robot" - pubsub = "service-%s@gcp-sa-pubsub" - secretmanager = "service-%s@gcp-sa-secretmanager" - storage = "service-%s@gs-project-accounts" + # TODO: deprecate gcf + gcf = "service-%s@gcf-admin-robot" + pubsub = "service-%s@gcp-sa-pubsub" + secretmanager = "service-%s@gcp-sa-secretmanager" + storage = "service-%s@gs-project-accounts" } service_accounts_default = { compute = "${local.project.number}-compute@developer.gserviceaccount.com" diff --git a/modules/project/shared-vpc.tf b/modules/project/shared-vpc.tf index ecce0df5..d228d87d 100644 --- a/modules/project/shared-vpc.tf +++ b/modules/project/shared-vpc.tf @@ -16,19 +16,41 @@ # tfdoc:file:description Shared VPC project-level configuration. +locals { + # compute the host project IAM bindings for this project's service identities + _svpc_service_identity_iam = coalesce( + local.svpc_service_config.service_identity_iam, {} + ) + _svpc_service_iam = flatten([ + for role, services in local._svpc_service_identity_iam : [ + for service in services : { role = role, service = service } + ] + ]) + svpc_host_config = { + enabled = coalesce( + try(var.shared_vpc_host_config.enabled, null), false + ) + service_projects = coalesce( + try(var.shared_vpc_host_config.service_projects, null), [] + ) + } + svpc_service_config = coalesce(var.shared_vpc_service_config, { + host_project = null, service_identity_iam = {} + }) + svpc_service_iam = { + for b in local._svpc_service_iam : "${b.role}:${b.service}" => b + } +} + resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { provider = google-beta - count = try(var.shared_vpc_host_config.enabled, false) ? 1 : 0 + count = local.svpc_host_config.enabled ? 1 : 0 project = local.project.project_id } resource "google_compute_shared_vpc_service_project" "service_projects" { - provider = google-beta - for_each = ( - try(var.shared_vpc_host_config.enabled, false) - ? toset(coalesce(var.shared_vpc_host_config.service_projects, [])) - : toset([]) - ) + provider = google-beta + for_each = toset(local.svpc_host_config.service_projects) host_project = local.project.project_id service_project = each.value depends_on = [google_compute_shared_vpc_host_project.shared_vpc_host] @@ -36,7 +58,19 @@ resource "google_compute_shared_vpc_service_project" "service_projects" { resource "google_compute_shared_vpc_service_project" "shared_vpc_service" { provider = google-beta - count = try(var.shared_vpc_service_config.attach, false) ? 1 : 0 + count = local.svpc_service_config.host_project != null ? 1 : 0 host_project = var.shared_vpc_service_config.host_project service_project = local.project.project_id } + +resource "google_project_iam_member" "shared_vpc_host_robots" { + for_each = local.svpc_service_iam + project = var.shared_vpc_service_config.host_project + role = each.value.role + member = ( + each.value.service == "cloudservices" + ? local.service_account_cloud_services + : local.service_accounts_robots[each.value.service] + ) +} + diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 3f81fbf2..7aa291f5 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -233,24 +233,17 @@ variable "shared_vpc_host_config" { enabled = bool service_projects = list(string) }) - default = { - enabled = false - service_projects = [] - } - nullable = false + default = null } variable "shared_vpc_service_config" { description = "Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config)." + # the list of valid service identities is in service-accounts.tf type = object({ - attach = bool - host_project = string + host_project = string + service_identity_iam = map(list(string)) }) - default = { - attach = false - host_project = "" - } - nullable = false + default = null } variable "skip_delete" { diff --git a/tests/examples/factories/project_factory/__init__.py b/tests/examples/factories/project_factory/__init__.py new file mode 100644 index 00000000..6d6d1266 --- /dev/null +++ b/tests/examples/factories/project_factory/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/examples/factories/project_factory/fixture/defaults.yaml b/tests/examples/factories/project_factory/fixture/defaults.yaml new file mode 100644 index 00000000..dc5b1616 --- /dev/null +++ b/tests/examples/factories/project_factory/fixture/defaults.yaml @@ -0,0 +1,24 @@ +# skip boilerplate check + +billing_account_id: 012345-67890A-BCDEF0 + +# [opt] Setup for billing alerts +billing_alert: + amount: 1000 + thresholds: + current: [0.5, 0.8] + forecasted: [0.5, 0.8] + credit_treatment: INCLUDE_ALL_CREDITS + +# [opt] Contacts for billing alerts and important notifications +essential_contacts: ["team-contacts@example.com"] + +# [opt] Labels set for all projects +labels: + environment: prod + department: accounting + application: example-app + foo: bar + +# [opt] Additional notification channels for billing +notification_channels: [] diff --git a/tests/examples/factories/project_factory/fixture/main.tf b/tests/examples/factories/project_factory/fixture/main.tf new file mode 100644 index 00000000..f81ab9f0 --- /dev/null +++ b/tests/examples/factories/project_factory/fixture/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _defaults = yamldecode(file(var.defaults_file)) + _defaults_net = { + billing_account_id = var.billing_account_id + environment_dns_zone = var.environment_dns_zone + shared_vpc_self_link = var.shared_vpc_self_link + vpc_host_project = var.vpc_host_project + } + defaults = merge(local._defaults, local._defaults_net) + projects = { + for f in fileset("${var.data_dir}", "**/*.yaml") : + trimsuffix(f, ".yaml") => yamldecode(file("${var.data_dir}/${f}")) + } +} + +module "projects" { + source = "../../../../../examples/factories/project-factory" + for_each = local.projects + defaults = local.defaults + project_id = each.key + billing_account_id = try(each.value.billing_account_id, null) + billing_alert = try(each.value.billing_alert, null) + dns_zones = try(each.value.dns_zones, []) + essential_contacts = try(each.value.essential_contacts, []) + folder_id = each.value.folder_id + group_iam = try(each.value.group_iam, {}) + iam = try(each.value.iam, {}) + kms_service_agents = try(each.value.kms, {}) + labels = try(each.value.labels, {}) + org_policies = try(each.value.org_policies, null) + service_accounts = try(each.value.service_accounts, {}) + services = try(each.value.services, []) + service_identities_iam = try(each.value.service_identities_iam, {}) + vpc = try(each.value.vpc, null) +} diff --git a/tests/examples/factories/project_factory/fixture/projects/project.yaml b/tests/examples/factories/project_factory/fixture/projects/project.yaml new file mode 100644 index 00000000..7ad16016 --- /dev/null +++ b/tests/examples/factories/project_factory/fixture/projects/project.yaml @@ -0,0 +1,100 @@ +# skip boilerplate check + +# [opt] Billing account id - overrides default if set +billing_account_id: 012345-67890A-BCDEF0 + +# [opt] Billing alerts config - overrides default if set +billing_alert: + amount: 10 + thresholds: + current: + - 0.5 + - 0.8 + forecasted: [] + credit_treatment: INCLUDE_ALL_CREDITS + +# [opt] DNS zones to be created as children of the environment_dns_zone defined in defaults +dns_zones: + - lorem + - ipsum + +# [opt] Contacts for billing alerts and important notifications +essential_contacts: + - team-a-contacts@example.com + +# Folder the project will be created as children of +folder_id: folders/012345678901 + +# [opt] Authoritative IAM bindings in group => [roles] format +group_iam: + test-team-foobar@fast-lab-0.gcp-pso-italy.net: + - roles/compute.admin + +# [opt] Authoritative IAM bindings in role => [principals] format +# Generally used to grant roles to service accounts external to the project +iam: + roles/compute.admin: + - serviceAccount:service-account + +# [opt] Service robots and keys they will be assigned as cryptoKeyEncrypterDecrypter +# in service => [keys] format +kms_service_agents: + compute: [key1, key2] + storage: [key1, key2] + +# [opt] Labels for the project - merged with the ones defined in defaults +labels: + environment: prod + +# [opt] Org policy overrides defined at project level +org_policies: + policy_boolean: + constraints/compute.disableGuestAttributesAccess: true + policy_list: + constraints/compute.trustedImageProjects: + inherit_from_parent: null + status: true + suggested_value: null + values: + - projects/fast-prod-iac-core-0 + +# [opt] Service account to create for the project and their roles on the project +# in name => [roles] format +service_accounts: + another-service-account: + - roles/compute.admin + my-service-account: + - roles/compute.admin + +# [opt] APIs to enable on the project. +services: + - storage.googleapis.com + - stackdriver.googleapis.com + - compute.googleapis.com + +# [opt] Roles to assign to the service identities in service => [roles] format +service_identities_iam: + compute: + - roles/storage.objectViewer + + # [opt] VPC setup. + # If set enables the `compute.googleapis.com` service and configures + # service project attachment +vpc: + # [opt] If set, enables the container API + gke_setup: + # Grants "roles/container.hostServiceAgentUser" to the container robot if set + enable_host_service_agent: false + + # Grants "roles/compute.securityAdmin" to the container robot if set + enable_security_admin: true + + # Host project the project will be service project of + host_project: fast-prod-net-spoke-0 + + # [opt] Subnets in the host project where principals will be granted networkUser + # in region/subnet-name => [principals] + subnets_iam: + europe-west1/prod-default-ew1: + - user:foobar@example.com + - serviceAccount:service-account1 diff --git a/tests/examples/factories/project_factory/fixture/variables.tf b/tests/examples/factories/project_factory/fixture/variables.tf new file mode 100644 index 00000000..0662bf78 --- /dev/null +++ b/tests/examples/factories/project_factory/fixture/variables.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "billing_account_id" { + description = "Billing account id." + type = string + default = "012345-67890A-BCDEF0" +} + +variable "data_dir" { + description = "Relative path for the folder storing configuration data." + type = string + default = "./projects/" +} + +variable "environment_dns_zone" { + description = "DNS zone suffix for environment." + type = string + default = "prod.gcp.example.com" +} + +variable "defaults_file" { + description = "Relative path for the file storing the project factory configuration." + type = string + default = "./defaults.yaml" +} + +variable "shared_vpc_self_link" { + description = "Self link for the shared VPC." + type = string + default = "self-link" +} + +variable "vpc_host_project" { + # tfdoc:variable:source 02-networking + description = "Host project for the shared VPC." + type = string + default = "host-project" +} diff --git a/tests/examples/factories/project_factory/test_plan.py b/tests/examples/factories/project_factory/test_plan.py new file mode 100644 index 00000000..f609b214 --- /dev/null +++ b/tests/examples/factories/project_factory/test_plan.py @@ -0,0 +1,18 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def test_counts(e2e_plan_runner): + "Check for a clean plan" + modules, resources = e2e_plan_runner() + assert len(modules) > 0 and len(resources) > 0 diff --git a/tests/examples/networking/filtering_proxy/test_plan.py b/tests/examples/networking/filtering_proxy/test_plan.py index 16a33a22..c13a5d1a 100644 --- a/tests/examples/networking/filtering_proxy/test_plan.py +++ b/tests/examples/networking/filtering_proxy/test_plan.py @@ -16,8 +16,8 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner() assert len(modules) == 11 - assert len(resources) == 29 + assert len(resources) == 30 modules, resources = e2e_plan_runner(mig="true") assert len(modules) == 13 - assert len(resources) == 35 + assert len(resources) == 36 diff --git a/tests/examples/networking/shared_vpc_gke/test_plan.py b/tests/examples/networking/shared_vpc_gke/test_plan.py index bc0aa921..8d0f6bd6 100644 --- a/tests/examples/networking/shared_vpc_gke/test_plan.py +++ b/tests/examples/networking/shared_vpc_gke/test_plan.py @@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner() assert len(modules) == 10 - assert len(resources) == 41 + assert len(resources) == 43 From c6a8f86cc04fcb1e8a0377036564d013aae39a38 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 9 Feb 2022 11:08:04 +0100 Subject: [PATCH 2/6] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4361e8e9..93939355 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ All notable changes to this project will be documented in this file. - remove GCS to BQ with Dataflow example, replace by GCS to BQ with least privileges - the `net-vpc` and `project` modules now use the beta provider for shared VPC-related resources - new `iot-core` module +- **incompatible change** the variables for host and service Shared VPCs have changed in the project module +- **incompatible change** the variable for service identities IAM has changed in the project factory ## [13.0.0] - 2022-01-27 From a18bea7f2ccb95b05e30b0820d600d92b4ac7380 Mon Sep 17 00:00:00 2001 From: eeaton Date: Wed, 9 Feb 2022 11:21:43 +0000 Subject: [PATCH 3/6] Add Zonal DNS Only org policy (#527) This is a safe and sane org policy that should be recommended for most customers to prevent them from accidentally configuring internal dns in a way that has reduced availability https://cloud.google.com/compute/docs/internal-dns#enforce_dns_by_policy --- fast/stages/01-resman/organization.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/fast/stages/01-resman/organization.tf b/fast/stages/01-resman/organization.tf index 5e3b072d..7ab1780e 100644 --- a/fast/stages/01-resman/organization.tf +++ b/fast/stages/01-resman/organization.tf @@ -87,6 +87,7 @@ module "organization" { "constraints/compute.requireOsLogin" = true "constraints/compute.restrictXpnProjectLienRemoval" = true "constraints/compute.skipDefaultNetworkCreation" = true + "constraints/compute.setNewProjectDefaultToZonalDNSOnly" = true "constraints/iam.automaticIamGrantsForDefaultServiceAccounts" = true "constraints/iam.disableServiceAccountKeyCreation" = true "constraints/iam.disableServiceAccountKeyUpload" = true From f4ef54da34c899bf6f6495486e4a5a434815a827 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 9 Feb 2022 12:53:17 +0100 Subject: [PATCH 4/6] Avoid nested tmp dirs in doc example tests (#528) --- tests/conftest.py | 7 +++++-- tests/doc_examples/test_plan.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8a81ed2e..9dc566b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -88,12 +88,15 @@ def e2e_plan_runner(_plan_runner): @ pytest.fixture(scope='session') -def example_plan_runner(_plan_runner): +def doc_example_plan_runner(_plan_runner): "Returns a function to run Terraform plan on documentation examples." def run_plan(fixture_path=None): "Runs Terraform plan and returns count of modules and resources." - plan = _plan_runner(fixture_path) + tf = tftest.TerraformTest(fixture_path, BASEDIR, + os.environ.get('TERRAFORM', 'terraform')) + tf.setup(upgrade=True) + plan = tf.plan(output=True, refresh=True) # the fixture is the example we are testing modules = plan.modules or {} return ( diff --git a/tests/doc_examples/test_plan.py b/tests/doc_examples/test_plan.py index 3aa1e8b7..0287a9d6 100644 --- a/tests/doc_examples/test_plan.py +++ b/tests/doc_examples/test_plan.py @@ -20,7 +20,7 @@ BASE_PATH = Path(__file__).parent EXPECTED_RESOURCES_RE = re.compile(r'# tftest modules=(\d+) resources=(\d+)') -def test_example(example_plan_runner, tmp_path, example): +def test_example(doc_example_plan_runner, tmp_path, example): (tmp_path / 'modules').symlink_to( Path(BASE_PATH, '../../modules/').resolve()) (tmp_path / 'variables.tf').symlink_to( @@ -31,6 +31,6 @@ def test_example(example_plan_runner, tmp_path, example): expected_modules = int(match.group(1)) if match is not None else 1 expected_resources = int(match.group(2)) if match is not None else 1 - num_modules, num_resources = example_plan_runner(str(tmp_path)) + num_modules, num_resources = doc_example_plan_runner(str(tmp_path)) assert expected_modules == num_modules assert expected_resources == num_resources From 307c29d2f89c3e0856e59018152033e434b51ace Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 9 Feb 2022 13:05:27 +0100 Subject: [PATCH 5/6] Add Data Catalog Policy Tag (#520) * First commit * Add outputs, update README, fix variables * Fix * Fix * Fix * Fix * Fix * Fix tests, for real? * Fix tests, for real. Co-authored-by: Ludovico Magnocavallo --- CHANGELOG.md | 2 + README.md | 2 +- modules/README.md | 1 + modules/data-catalog-policy-tag/README.md | 62 ++++++++++++++++ modules/data-catalog-policy-tag/iam.tf | 67 +++++++++++++++++ modules/data-catalog-policy-tag/main.tf | 43 +++++++++++ modules/data-catalog-policy-tag/outputs.tf | 25 +++++++ modules/data-catalog-policy-tag/variables.tf | 78 ++++++++++++++++++++ modules/data-catalog-policy-tag/versions.tf | 29 ++++++++ 9 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 modules/data-catalog-policy-tag/README.md create mode 100644 modules/data-catalog-policy-tag/iam.tf create mode 100644 modules/data-catalog-policy-tag/main.tf create mode 100644 modules/data-catalog-policy-tag/outputs.tf create mode 100644 modules/data-catalog-policy-tag/variables.tf create mode 100644 modules/data-catalog-policy-tag/versions.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 93939355..1d1f83c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file. - new `iot-core` module - **incompatible change** the variables for host and service Shared VPCs have changed in the project module - **incompatible change** the variable for service identities IAM has changed in the project factory +- the `net-vpc` and `project` modules now use the beta provider for shared VPC-related resources +- add `data-catalog-policy-tag` module ## [13.0.0] - 2022-01-27 diff --git a/README.md b/README.md index 782fd5bc..609c58ff 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Currently available modules: - **foundational** - [folder](./modules/folder), [organization](./modules/organization), [project](./modules/project), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [billing budget](./modules/billing-budget), [naming convention](./modules/naming-convention) - **networking** - [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN static](./modules/net-vpn-static), [VPN dynamic](./modules/net-vpn-dynamic), [VPN HA](./modules/net-vpn-ha), [NAT](./modules/net-cloudnat), [address reservation](./modules/net-address), [DNS](./modules/dns), [L4 ILB](./modules/net-ilb), [Service Directory](./modules/service-directory), [Cloud Endpoints](./modules/endpoints) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [GKE cluster](./modules/gke-cluster), [GKE nodepool](./modules/gke-nodepool), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid) -- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion), [Bigtable instance](./modules/bigtable-instance), [Cloud SQL instance](./modules/cloudsql-instance) +- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion), [Bigtable instance](./modules/bigtable-instance), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag) - **development** - [Cloud Source Repository](./modules/source-repository), [Container Registry](./modules/container-registry), [Artifact Registry](./modules/artifact-registry), [Apigee Organization](./modules/apigee-organization), [Apigee X Instance](./modules/apigee-x-instance) - **security** - [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc) - **serverless** - [Cloud Function](./modules/cloud-function), [Cloud Run](./modules/cloud-run) diff --git a/modules/README.md b/modules/README.md index 9b1913c7..f22878bf 100644 --- a/modules/README.md +++ b/modules/README.md @@ -69,6 +69,7 @@ These modules are used in the examples included in this repository. If you are u - [Pub/Sub](./pubsub) - [Bigtable instance](./bigtable-instance) - [Cloud SQL instance](./cloudsql-instance) +- [Data Catalog Policy Tag](./data-catalog-policy-tag) ## Development diff --git a/modules/data-catalog-policy-tag/README.md b/modules/data-catalog-policy-tag/README.md new file mode 100644 index 00000000..1ffd6a26 --- /dev/null +++ b/modules/data-catalog-policy-tag/README.md @@ -0,0 +1,62 @@ +# Data Catalog Module + +This module simplifies the creation of [Data Catalog](https://cloud.google.com/data-catalog) Policy Tags. Policy Tags can be used to configure [Bigquery column-level access](https://cloud.google.com/bigquery/docs/best-practices-policy-tags). + +Note: Data Catalog is still in beta, hence this module currently uses the beta provider. +## Examples + +### Simple Taxonomy with policy tags + +```hcl +module "cmn-dc" { + source = "./modules/data-catalog-policy-tag" + name = "my-datacatalog-policy-tags" + project_id = "my-project" + tags = ["low", "medium", "high"] +} +# tftest modules=1 resources=4 +``` + +### Simple Taxonomy with IAM binding + +```hcl +module "cmn-dc" { + source = "./modules/data-catalog-policy-tag" + name = "my-datacatalog-policy-tags" + project_id = "my-project" + tags = ["low", "medium", "high"] + iam = { + "roles/datacatalog.categoryAdmin" = ["group:GROUP_NAME@example.com"] + } +} +# tftest modules=1 resources=5 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L59) | Name of this taxonomy. | string | ✓ | | +| [project_id](variables.tf#L70) | GCP project id. | | ✓ | | +| [activated_policy_types](variables.tf#L17) | A list of policy types that are activated for this taxonomy. | list(string) | | ["FINE_GRAINED_ACCESS_CONTROL"] | +| [description](variables.tf#L23) | Description of this taxonomy. | string | | "Taxonomy - Terraform managed" | +| [group_iam](variables.tf#L29) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L35) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive](variables.tf#L41) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive_members](variables.tf#L47) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | +| [location](variables.tf#L53) | Data Catalog Taxonomy location. | string | | "eu" | +| [prefix](variables.tf#L64) | Prefix used to generate project id and name. | string | | null | +| [tags](variables.tf#L74) | List of Data Catalog Policy tags to be created. | list(string) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [tags](outputs.tf#L17) | Policy Tags. | | +| [taxonomy_id](outputs.tf#L22) | Taxonomy id. | | + + +## TODO +- Support IAM at tag level. +- Support Child policy tags \ No newline at end of file diff --git a/modules/data-catalog-policy-tag/iam.tf b/modules/data-catalog-policy-tag/iam.tf new file mode 100644 index 00000000..b7e50c0e --- /dev/null +++ b/modules/data-catalog-policy-tag/iam.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Data Catalog Taxonomy IAM definition. + +locals { + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _iam_additive_member_pairs = flatten([ + for member, roles in var.iam_additive_members : [ + for role in roles : { role = role, member = member } + ] + ]) + _iam_additive_pairs = flatten([ + for role, members in var.iam_additive : [ + for member in members : { role = role, member = member } + ] + ]) + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } + iam_additive = { + for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) : + "${pair.role}-${pair.member}" => pair + } +} + +resource "google_data_catalog_taxonomy_iam_binding" "authoritative" { + provider = google-beta + for_each = local.iam + role = each.key + members = each.value + taxonomy = google_data_catalog_taxonomy.default.id +} + +resource "google_data_catalog_taxonomy_iam_member" "additive" { + provider = google-beta + for_each = ( + length(var.iam_additive) + length(var.iam_additive_members) > 0 + ? local.iam_additive + : {} + ) + role = each.value.role + member = each.value.member + taxonomy = google_data_catalog_taxonomy.default.id +} diff --git a/modules/data-catalog-policy-tag/main.tf b/modules/data-catalog-policy-tag/main.tf new file mode 100644 index 00000000..536b40b5 --- /dev/null +++ b/modules/data-catalog-policy-tag/main.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Data Catalog Taxonomy definition + +locals { + name = ( + var.name != null ? var.name : "${local.prefix}taxonomy" + ) + prefix = var.prefix == null ? "" : "${var.prefix}-" +} + +resource "google_data_catalog_taxonomy" "default" { + provider = google-beta + project = var.project_id + region = var.location + display_name = local.name + description = var.description + activated_policy_types = var.activated_policy_types +} + +resource "google_data_catalog_policy_tag" "default" { + for_each = toset(var.tags) + provider = google-beta + taxonomy = google_data_catalog_taxonomy.default.id + display_name = each.key + description = "${each.key} - Terraform managed. " +} + +#TODO Add IAM at tag level diff --git a/modules/data-catalog-policy-tag/outputs.tf b/modules/data-catalog-policy-tag/outputs.tf new file mode 100644 index 00000000..fcd5cc29 --- /dev/null +++ b/modules/data-catalog-policy-tag/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "tags" { + description = "Policy Tags." + value = { for k, v in google_data_catalog_policy_tag.default : v.id => v.name } +} + +output "taxonomy_id" { + description = "Taxonomy id." + value = google_data_catalog_taxonomy.default.id +} diff --git a/modules/data-catalog-policy-tag/variables.tf b/modules/data-catalog-policy-tag/variables.tf new file mode 100644 index 00000000..92cb6b38 --- /dev/null +++ b/modules/data-catalog-policy-tag/variables.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "activated_policy_types" { + description = "A list of policy types that are activated for this taxonomy." + type = list(string) + default = ["FINE_GRAINED_ACCESS_CONTROL"] +} + +variable "description" { + description = "Description of this taxonomy." + type = string + default = "Taxonomy - Terraform managed" +} + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "iam_additive" { + description = "IAM additive bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "iam_additive_members" { + description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values." + type = map(list(string)) + default = {} +} + +variable "location" { + description = "Data Catalog Taxonomy location." + type = string + default = "eu" +} + +variable "name" { + description = "Name of this taxonomy." + type = string +} + +variable "prefix" { + description = "Prefix used to generate project id and name." + type = string + default = null +} + +variable "project_id" { + description = "GCP project id." +} + +variable "tags" { + description = "List of Data Catalog Policy tags to be created." + type = list(string) + default = [] +} diff --git a/modules/data-catalog-policy-tag/versions.tf b/modules/data-catalog-policy-tag/versions.tf new file mode 100644 index 00000000..e72a7800 --- /dev/null +++ b/modules/data-catalog-policy-tag/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.0.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.0.0" + } + } +} + + From b377b3091060806869701c7684bd4efd8b7f37c4 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 9 Feb 2022 13:05:46 +0100 Subject: [PATCH 6/6] Update README (#517) Update path to config folder Co-authored-by: Ludovico Magnocavallo --- fast/stages/03-project-factory/prod/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fast/stages/03-project-factory/prod/README.md b/fast/stages/03-project-factory/prod/README.md index 2b31523b..9e29742f 100644 --- a/fast/stages/03-project-factory/prod/README.md +++ b/fast/stages/03-project-factory/prod/README.md @@ -56,7 +56,7 @@ It's of course possible to run this stage in isolation, by making sure the archi If you're running this on top of Fast, you should run the following commands to create the providers file, and populate the required variables from the previous stage. ```bash -# Variable `outputs_location` is set to `../../config` in stage 01-resman +# Variable `outputs_location` is set to `../../../config` in stage 01-resman $ cd fabric-fast/stages/03-project-factory/prod ln -s ../../../config/03-project-factory-prod/providers.tf ``` @@ -73,7 +73,7 @@ To avoid the tedious job of filling in the first group of variables with values If you configured a valid path for `outputs_location` in the bootstrap and networking stage, simply link the relevant `terraform-*.auto.tfvars.json` files from this stage's outputs folder (under the path you specified), where the `*` above is set to the name of the stage that produced it. For this stage, a single `.tfvars` file is available: ```bash -# Variable `outputs_location` is set to `../../config` in stages 01-bootstrap and the 02-networking stage in use +# Variable `outputs_location` is set to `../../../config` in stages 01-bootstrap and the 02-networking stage in use ln -s ../../../config/03-project-factory-prod/terraform-bootstrap.auto.tfvars.json ln -s ../../../config/03-project-factory-prod/terraform-networking.auto.tfvars.json ```