From e77acf621a6d24917ddc610170e4b86124c7686c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 2 Nov 2019 18:30:52 +0100 Subject: [PATCH 1/7] fix service dependency and number of services in shared vpc sample --- infrastructure/shared-vpc/main.tf | 30 ++++++++++++++++++-------- infrastructure/shared-vpc/variables.tf | 19 ++-------------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/infrastructure/shared-vpc/main.tf b/infrastructure/shared-vpc/main.tf index ed37c5e8..a2a84d1b 100644 --- a/infrastructure/shared-vpc/main.tf +++ b/infrastructure/shared-vpc/main.tf @@ -19,21 +19,26 @@ # host project module "project-svpc-host" { - source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + # source = "terraform-google-modules/project-factory/google//modules/fabric-project" + # version = "3.3.1" + source = "github.com/terraform-google-modules/terraform-google-project-factory//modules/fabric-project" parent = var.root_node prefix = var.prefix name = "vpc-host" billing_account = var.billing_account_id owners = var.owners_host - activate_apis = var.project_services + activate_apis = concat( + var.project_services, + ["dns.googleapis.com", "cloudkms.googleapis.com"] + ) } # service projects module "project-service-gce" { - source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + # source = "terraform-google-modules/project-factory/google//modules/fabric-project" + # version = "3.3.1" + source = "github.com/terraform-google-modules/terraform-google-project-factory//modules/fabric-project" parent = var.root_node prefix = var.prefix name = "gce" @@ -42,18 +47,25 @@ module "project-service-gce" { owners = var.owners_gce oslogin_admins = var.oslogin_admins_gce oslogin_users = var.oslogin_users_gce - activate_apis = var.project_services + activate_apis = concat( + var.project_services, + ["container.googleapis.com"] + ) } module "project-service-gke" { - source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + # source = "terraform-google-modules/project-factory/google//modules/fabric-project" + # version = "3.3.1" + source = "github.com/terraform-google-modules/terraform-google-project-factory//modules/fabric-project" parent = var.root_node prefix = var.prefix name = "gke" billing_account = var.billing_account_id owners = var.owners_gke - activate_apis = var.project_services + activate_apis = concat( + var.project_services, + ["resourceviews.googleapis.com"] + ) } ################################################################################ diff --git a/infrastructure/shared-vpc/variables.tf b/infrastructure/shared-vpc/variables.tf index 89fb92ce..e3f325e7 100644 --- a/infrastructure/shared-vpc/variables.tf +++ b/infrastructure/shared-vpc/variables.tf @@ -107,25 +107,10 @@ variable "subnet_secondary_ranges" { variable "project_services" { description = "Service APIs enabled by default in new projects." default = [ - "bigquery-json.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudbilling.googleapis.com", - "cloudkms.googleapis.com", - "cloudresourcemanager.googleapis.com", + "bigquery.googleapis.com", "compute.googleapis.com", - "container.googleapis.com", - "containerregistry.googleapis.com", - "deploymentmanager.googleapis.com", - "dns.googleapis.com", "iam.googleapis.com", - "iamcredentials.googleapis.com", - "logging.googleapis.com", - "oslogin.googleapis.com", - "pubsub.googleapis.com", - "replicapool.googleapis.com", - "replicapoolupdater.googleapis.com", - "resourceviews.googleapis.com", - "serviceusage.googleapis.com", + "stackdriver.googleapis.com", "storage-api.googleapis.com", ] } From 738d613126449d8d8adebff37f2ddaf8a1639f4d Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 3 Nov 2019 09:26:28 +0100 Subject: [PATCH 2/7] fix shared vpc project services --- infrastructure/shared-vpc/main.tf | 13 ++++--------- infrastructure/shared-vpc/variables.tf | 5 +---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/infrastructure/shared-vpc/main.tf b/infrastructure/shared-vpc/main.tf index a2a84d1b..9fb319ff 100644 --- a/infrastructure/shared-vpc/main.tf +++ b/infrastructure/shared-vpc/main.tf @@ -47,10 +47,7 @@ module "project-service-gce" { owners = var.owners_gce oslogin_admins = var.oslogin_admins_gce oslogin_users = var.oslogin_users_gce - activate_apis = concat( - var.project_services, - ["container.googleapis.com"] - ) + activate_apis = var.project_services } module "project-service-gke" { @@ -62,10 +59,7 @@ module "project-service-gke" { name = "gke" billing_account = var.billing_account_id owners = var.owners_gke - activate_apis = concat( - var.project_services, - ["resourceviews.googleapis.com"] - ) + activate_apis = var.project_services } ################################################################################ @@ -117,7 +111,8 @@ module "net-svpc-access" { host_project_id = module.project-svpc-host.project_id service_project_num = 2 service_project_ids = [ - module.project-service-gce.project_id, module.project-service-gke.project_id + module.project-service-gce.project_id, + module.project-service-gke.project_id ] host_subnets = ["gce", "gke"] host_subnet_regions = compact([ diff --git a/infrastructure/shared-vpc/variables.tf b/infrastructure/shared-vpc/variables.tf index e3f325e7..0efe0bab 100644 --- a/infrastructure/shared-vpc/variables.tf +++ b/infrastructure/shared-vpc/variables.tf @@ -107,10 +107,7 @@ variable "subnet_secondary_ranges" { variable "project_services" { description = "Service APIs enabled by default in new projects." default = [ - "bigquery.googleapis.com", - "compute.googleapis.com", - "iam.googleapis.com", + "resourceviews.googleapis.com", "stackdriver.googleapis.com", - "storage-api.googleapis.com", ] } From 91513025e2bc77fecab9200c18ec10e692c9c4de Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 4 Nov 2019 20:44:15 +0100 Subject: [PATCH 3/7] foundations env: update module versions --- foundations/environments/main.tf | 18 ++++++++++-------- foundations/environments/variables.tf | 18 +----------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/foundations/environments/main.tf b/foundations/environments/main.tf index e42afbc5..d96745cc 100644 --- a/foundations/environments/main.tf +++ b/foundations/environments/main.tf @@ -20,7 +20,7 @@ module "project-tf" { source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + version = "5.0.0" parent = var.root_node billing_account = var.billing_account_id prefix = var.prefix @@ -34,7 +34,7 @@ module "project-tf" { module "service-accounts-tf-environments" { source = "terraform-google-modules/service-accounts/google" - version = "2.0.0" + version = "2.0.1" project_id = module.project-tf.project_id org_id = var.organization_id billing_account_id = var.billing_account_id @@ -102,21 +102,23 @@ module "folders-top-level" { module "project-audit" { source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + version = "5.0.0" parent = var.root_node billing_account = var.billing_account_id prefix = var.prefix name = "audit" lien_reason = "audit" - activate_apis = var.project_services - viewers = var.audit_viewers + activate_apis = concat(var.project_services, [ + "bigquery.googleapis.com", + ]) + viewers = var.audit_viewers } # audit logs destination on BigQuery module "bq-audit-export" { source = "terraform-google-modules/log-export/google//modules/bigquery" - version = "3.0.0" + version = "3.1.0" project_id = module.project-audit.project_id dataset_name = "logs_audit_${replace(var.environments[0], "-", "_")}" log_sink_writer_identity = module.log-sink-audit.writer_identity @@ -127,7 +129,7 @@ module "bq-audit-export" { module "log-sink-audit" { source = "terraform-google-modules/log-export/google" - version = "3.0.0" + version = "3.1.0" filter = "logName: \"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName: \"/logs/cloudaudit.googleapis.com%2Fsystem_event\"" log_sink_name = "logs-audit-${var.environments[0]}" parent_resource_type = "folder" @@ -146,7 +148,7 @@ module "log-sink-audit" { module "project-shared-resources" { source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + version = "5.0.0" parent = var.root_node billing_account = var.billing_account_id prefix = var.prefix diff --git a/foundations/environments/variables.tf b/foundations/environments/variables.tf index b3dd4d9d..2b8e6f09 100644 --- a/foundations/environments/variables.tf +++ b/foundations/environments/variables.tf @@ -81,23 +81,7 @@ variable "terraform_owners" { variable "project_services" { description = "Service APIs enabled by default in new projects." default = [ - "bigquery-json.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudbilling.googleapis.com", - "cloudresourcemanager.googleapis.com", - "compute.googleapis.com", - "container.googleapis.com", - "containerregistry.googleapis.com", - "deploymentmanager.googleapis.com", - "iam.googleapis.com", - "iamcredentials.googleapis.com", - "logging.googleapis.com", - "oslogin.googleapis.com", - "pubsub.googleapis.com", - "replicapool.googleapis.com", - "replicapoolupdater.googleapis.com", "resourceviews.googleapis.com", - "serviceusage.googleapis.com", - "storage-api.googleapis.com", + "stackdriver.googleapis.com", ] } From f6370df9389e93709cdf08bf5cf3d68c6bccf6a6 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 4 Nov 2019 20:44:53 +0100 Subject: [PATCH 4/7] infra svpc: update module versions --- infrastructure/shared-vpc/main.tf | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/infrastructure/shared-vpc/main.tf b/infrastructure/shared-vpc/main.tf index 9fb319ff..4816f48f 100644 --- a/infrastructure/shared-vpc/main.tf +++ b/infrastructure/shared-vpc/main.tf @@ -19,9 +19,8 @@ # host project module "project-svpc-host" { - # source = "terraform-google-modules/project-factory/google//modules/fabric-project" - # version = "3.3.1" - source = "github.com/terraform-google-modules/terraform-google-project-factory//modules/fabric-project" + source = "terraform-google-modules/project-factory/google//modules/fabric-project" + version = "5.0.0" parent = var.root_node prefix = var.prefix name = "vpc-host" @@ -36,9 +35,8 @@ module "project-svpc-host" { # service projects module "project-service-gce" { - # source = "terraform-google-modules/project-factory/google//modules/fabric-project" - # version = "3.3.1" - source = "github.com/terraform-google-modules/terraform-google-project-factory//modules/fabric-project" + source = "terraform-google-modules/project-factory/google//modules/fabric-project" + version = "5.0.0" parent = var.root_node prefix = var.prefix name = "gce" @@ -51,9 +49,8 @@ module "project-service-gce" { } module "project-service-gke" { - # source = "terraform-google-modules/project-factory/google//modules/fabric-project" - # version = "3.3.1" - source = "github.com/terraform-google-modules/terraform-google-project-factory//modules/fabric-project" + source = "terraform-google-modules/project-factory/google//modules/fabric-project" + version = "5.0.0" parent = var.root_node prefix = var.prefix name = "gke" From 554af0e3d5b45907f58b2e264e6f5b48ab991ea0 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 4 Nov 2019 20:45:23 +0100 Subject: [PATCH 5/7] foundations env: fix project tests --- tests/foundations/environments/test_projects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/foundations/environments/test_projects.py b/tests/foundations/environments/test_projects.py index 68648351..26d30639 100644 --- a/tests/foundations/environments/test_projects.py +++ b/tests/foundations/environments/test_projects.py @@ -39,6 +39,6 @@ def test_project_services(plan, project_modules): "Project service resource must enable APIs specified in the variable." num_services = len(plan.variables['project_services']) for mod in project_modules.values(): - project_services = [r for r in mod.child_modules['module.project_services'].resources if r.startswith( + project_services = [r for r in mod.resources if r.startswith( 'google_project_service.project_services')] - assert len(project_services) == num_services + assert len(project_services) >= num_services From 2bb037f12a7d5c3304993a3fa5fe60c4428980ea Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 4 Nov 2019 20:50:08 +0100 Subject: [PATCH 6/7] foundations bu: update module versions --- foundations/business-units/main.tf | 25 ++++++++++++------------- foundations/business-units/variables.tf | 18 +----------------- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/foundations/business-units/main.tf b/foundations/business-units/main.tf index e0809d3c..3252e67c 100644 --- a/foundations/business-units/main.tf +++ b/foundations/business-units/main.tf @@ -39,7 +39,7 @@ module "shared-folder" { module "project-tf" { source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + version = "5.0.0" parent = module.shared-folder.id billing_account = var.billing_account_id prefix = var.prefix @@ -53,7 +53,7 @@ module "project-tf" { module "service-accounts-tf-environments" { source = "terraform-google-modules/service-accounts/google" - version = "2.0.0" + version = "2.0.1" project_id = module.project-tf.project_id org_id = var.organization_id billing_account_id = var.billing_account_id @@ -97,8 +97,7 @@ module "gcs-tf-environments" { # Business unit 1 module "business-unit-1-folders" { - source = "./modules/business-unit-folders" - + source = "./modules/business-unit-folders" business_unit_folder_name = var.business_unit_1_name environments = var.environments per_folder_admins = module.service-accounts-tf-environments.iam_emails_list @@ -109,8 +108,7 @@ module "business-unit-1-folders" { # Business unit 2 module "business-unit-2-folders" { - source = "./modules/business-unit-folders" - + source = "./modules/business-unit-folders" business_unit_folder_name = var.business_unit_2_name environments = var.environments per_folder_admins = module.service-accounts-tf-environments.iam_emails_list @@ -121,8 +119,7 @@ module "business-unit-2-folders" { # Business unit 3 module "business-unit-3-folders" { - source = "./modules/business-unit-folders" - + source = "./modules/business-unit-folders" business_unit_folder_name = var.business_unit_3_name environments = var.environments per_folder_admins = module.service-accounts-tf-environments.iam_emails_list @@ -138,21 +135,23 @@ module "business-unit-3-folders" { module "project-audit" { source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + version = "5.0.0" parent = module.shared-folder.id billing_account = var.billing_account_id prefix = var.prefix name = "audit" lien_reason = "audit" - activate_apis = var.project_services viewers = var.audit_viewers + activate_apis = concat(var.project_services, [ + "bigquery.googleapis.com", + ]) } # Audit logs destination on BigQuery module "bq-audit-export" { source = "terraform-google-modules/log-export/google//modules/bigquery" - version = "3.0.0" + version = "3.1.0" project_id = module.project-audit.project_id dataset_name = "${replace(local.log_sink_name, "-", "_")}" log_sink_writer_identity = module.log-sink-audit.writer_identity @@ -162,7 +161,7 @@ module "bq-audit-export" { module "log-sink-audit" { source = "terraform-google-modules/log-export/google" - version = "3.0.0" + version = "3.1.0" filter = "logName: \"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName: \"/logs/cloudaudit.googleapis.com%2Fsystem_event\"" log_sink_name = local.log_sink_name parent_resource_type = local.log_sink_parent_resource_type @@ -180,7 +179,7 @@ module "log-sink-audit" { module "project-shared-resources" { source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "3.3.1" + version = "5.0.0" parent = module.shared-folder.id billing_account = var.billing_account_id prefix = var.prefix diff --git a/foundations/business-units/variables.tf b/foundations/business-units/variables.tf index c3828356..c25a1755 100644 --- a/foundations/business-units/variables.tf +++ b/foundations/business-units/variables.tf @@ -86,23 +86,7 @@ variable "terraform_owners" { variable "project_services" { description = "Service APIs enabled by default in new projects." default = [ - "bigquery-json.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudbilling.googleapis.com", - "cloudresourcemanager.googleapis.com", - "compute.googleapis.com", - "container.googleapis.com", - "containerregistry.googleapis.com", - "deploymentmanager.googleapis.com", - "iam.googleapis.com", - "iamcredentials.googleapis.com", - "logging.googleapis.com", - "oslogin.googleapis.com", - "pubsub.googleapis.com", - "replicapool.googleapis.com", - "replicapoolupdater.googleapis.com", "resourceviews.googleapis.com", - "serviceusage.googleapis.com", - "storage-api.googleapis.com", + "stackdriver.googleapis.com", ] } From aacd679f26568013c72e1423fb73fa0ff425a5f9 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 4 Nov 2019 20:50:20 +0100 Subject: [PATCH 7/7] foundations bu: fix project tests --- tests/foundations/business-units/test_projects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/foundations/business-units/test_projects.py b/tests/foundations/business-units/test_projects.py index ba56a474..23194afa 100644 --- a/tests/foundations/business-units/test_projects.py +++ b/tests/foundations/business-units/test_projects.py @@ -41,6 +41,6 @@ def test_project_services(plan, project_modules): "Project service resource must enable APIs specified in the variable." num_services = len(plan.variables['project_services']) for mod in project_modules.values(): - project_services = [r for r in mod.child_modules['module.project_services'].resources if r.startswith( + project_services = [r for r in mod.resources if r.startswith( 'google_project_service.project_services')] - assert len(project_services) == num_services + assert len(project_services) >= num_services