Add support for Shared VPC service IAM to project module (#525)

* project module changes

* fix examples

* add comments in module code

* re-enable nullable on svpc variables

* project factory

* Tests still failing (#526)

* fix pf

* tfdoc

* pf test boilerplate

Co-authored-by: Simone Ruffilli <sruffilli@google.com>

examples/factories/project-factory/README.md

@@ -231,15 +231,15 @@ vpc:
 | [org_policies](variables.tf#L98) | Org-policy overrides at project level. | <code title="object&#40;&#123;&#10;  policy_boolean &#61; map&#40;bool&#41;&#10;  policy_list &#61; map&#40;object&#40;&#123;&#10;    inherit_from_parent &#61; bool&#10;    suggested_value     &#61; string&#10;    status              &#61; bool&#10;    values              &#61; list&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
 | [prefix](variables.tf#L112) | Prefix used for the project id. | <code>string</code> |  | <code>null</code> |
 | [service_accounts](variables.tf#L123) | Service accounts to be created, and roles to assign them. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [service_identities_iam](variables.tf#L136) | Custom IAM settings for service identities in service => [role] format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [services](variables.tf#L129) | Services to be enabled for the project. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
-| [services_iam](variables.tf#L135) | Custom IAM settings for robot ServiceAccounts in service => [role] format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [vpc](variables.tf#L141) | VPC configuration for the project. | <code title="object&#40;&#123;&#10;  host_project &#61; string&#10;  gke_setup &#61; object&#40;&#123;&#10;    enable_security_admin     &#61; bool&#10;    enable_host_service_agent &#61; bool&#10;  &#125;&#41;&#10;  subnets_iam &#61; map&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [vpc](variables.tf#L143) | VPC configuration for the project. | <code title="object&#40;&#123;&#10;  host_project &#61; string&#10;  gke_setup &#61; object&#40;&#123;&#10;    enable_security_admin     &#61; bool&#10;    enable_host_service_agent &#61; bool&#10;  &#125;&#41;&#10;  subnets_iam &#61; map&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
 
 ## Outputs
 
 | name | description | sensitive |
 |---|---|:---:|
 | [project](outputs.tf#L19) | The project resource as return by the `project` module |  |
-| [project_id](outputs.tf#L30) | Project ID. |  |
+| [project_id](outputs.tf#L29) | Project ID. |  |
 
 <!-- END TFDOC -->

examples/factories/project-factory/main.tf

@@ -15,24 +15,26 @@
  */
 
 locals {
-  _gke_iam_hsau = try(var.vpc.gke_setup.enable_host_service_agent, false) ? {
-    "roles/container.hostServiceAgentUser" = "serviceAccount:${module.project.service_accounts.robots.container-engine}"
-  } : {}
-  _gke_iam_securityadmin = try(var.vpc.gke_setup.enable_security_admin, false) ? {
-    "roles/compute.securityAdmin" = "serviceAccount:${module.project.service_accounts.robots.container-engine}"
-  } : {}
+  # internal structures for group IAM bindings
   _group_iam = {
-    for r in local._group_iam_roles : r => [
-      for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null
+    for r in local._group_iam_bindings : r => [
+      for k, v in var.group_iam :
+      "group:${k}" if try(index(v, r), null) != null
     ]
   }
-  _group_iam_roles = distinct(flatten(values(var.group_iam)))
+  _group_iam_bindings = distinct(flatten(values(var.group_iam)))
+  # internal structures for project service accounts IAM bindings
   _service_accounts_iam = {
-    for r in local._service_accounts_iam_roles : r => [
-      for k, v in var.service_accounts : "serviceAccount:${k}@${var.project_id}.iam.gserviceaccount.com" if try(index(v, r), null) != null
+    for r in local._service_accounts_iam_bindings : r => [
+      for k, v in var.service_accounts :
+      "serviceAccount:${k}@${var.project_id}.iam.gserviceaccount.com"
+      if try(index(v, r), null) != null
     ]
   }
-  _service_accounts_iam_roles = distinct(flatten(values(var.service_accounts)))
+  _service_accounts_iam_bindings = distinct(flatten(
+    values(var.service_accounts)
+  ))
+  # internal structures for project services
   _services = concat([
     "billingbudgets.googleapis.com",
     "essentialcontacts.googleapis.com"
@@ -41,46 +43,81 @@ locals {
     try(var.vpc.gke_setup, null) != null ? ["container.googleapis.com"] : [],
     var.vpc != null ? ["compute.googleapis.com"] : [],
   )
-  _services_iam_roles = distinct(flatten(values(var.services_iam)))
-  _services_iam = {
-    for r in local._services_iam_roles : r => [
-      for k, v in var.services_iam : "serviceAccount:${module.project.service_accounts.robots[k]}" if try(index(v, r), null) != null
+  # internal structures for service identity IAM bindings
+  _service_identities_roles = distinct(flatten(values(var.service_identities_iam)))
+  _service_identities_iam = {
+    for role in local._service_identities_roles : role => [
+      for service, roles in var.service_identities_iam :
+      "serviceAccount:${module.project.service_accounts.robots[service]}"
+      if contains(roles, role)
     ]
   }
-  billing_account_id = coalesce(var.billing_account_id, try(var.defaults.billing_account_id, ""))
-  billing_alert      = var.billing_alert == null ? try(var.defaults.billing_alert, null) : var.billing_alert
-  essential_contacts = concat(try(var.defaults.essential_contacts, []), var.essential_contacts)
-  host_project_bindings = merge(
-    local._gke_iam_hsau,
-    local._gke_iam_securityadmin
+  # internal structure for Shared VPC service project IAM bindings
+  _vpc_subnet_bindings = (
+    local.vpc.subnets_iam == null || local.vpc.host_project == null
+    ? []
+    : flatten([
+      for subnet, members in local.vpc.subnets_iam : [
+        for member in members : {
+          region = split("/", subnet)[0]
+          subnet = split("/", subnet)[1]
+          member = member
+        }
+      ]
+    ])
   )
+  # structures for billing id
+  billing_account_id = coalesce(
+    var.billing_account_id, try(var.defaults.billing_account_id, "")
+  )
+  billing_alert = (
+    var.billing_alert == null
+    ? try(var.defaults.billing_alert, null)
+    : var.billing_alert
+  )
+  # structure for essential contacts
+  essential_contacts = concat(
+    try(var.defaults.essential_contacts, []), var.essential_contacts
+  )
+  # structure that combines all authoritative IAM bindings
   iam = {
     for role in distinct(concat(
       keys(var.iam),
       keys(local._group_iam),
       keys(local._service_accounts_iam),
-      keys(local._services_iam),
+      keys(local._service_identities_iam),
     )) :
     role => concat(
       try(var.iam[role], []),
       try(local._group_iam[role], []),
       try(local._service_accounts_iam[role], []),
-      try(local._services_iam[role], []),
+      try(local._service_identities_iam[role], []),
     )
   }
-  labels = merge(coalesce(var.labels, {}), coalesce(try(var.defaults.labels, {}), {}))
-  network_user_service_accounts = concat(
-    contains(local.services, "compute.googleapis.com") ? [
-      "serviceAccount:${module.project.service_accounts.robots.compute}"
-    ] : [],
-    contains(local.services, "container.googleapis.com") ? [
-      "serviceAccount:${module.project.service_accounts.robots.container-engine}",
-      "serviceAccount:${module.project.service_accounts.cloud_services}"
-    ] : [],
-  [])
-  services         = distinct(concat(var.services, local._services))
-  vpc_host_project = try(var.vpc.host_project, var.defaults.vpc_host_project)
-  vpc_setup        = var.vpc != null
+  # merge labels with defaults
+  labels = merge(
+    coalesce(var.labels, {}), coalesce(try(var.defaults.labels, {}), {})
+  )
+  # deduplicate services
+  services = distinct(concat(var.services, local._services))
+  # structures for Shared VPC resources in host project
+  vpc = coalesce(var.vpc, {
+    host_project = null, gke_setup = null, subnets_iam = null
+  })
+  vpc_cloudservices = (
+    local.vpc_gke_service_agent ||
+    contains(var.services, "compute.googleapis.com")
+  )
+  vpc_gke_security_admin = coalesce(
+    try(local.vpc.gke_setup.enable_security_admin, null), false
+  )
+  vpc_gke_service_agent = coalesce(
+    try(local.vpc.gke_setup.enable_host_service_agent, null), false
+  )
+  vpc_subnet_bindings = {
+    for binding in local._vpc_subnet_bindings :
+    "${binding.subnet}:${binding.member}" => binding
+  }
 }
 
 module "billing-alert" {
@@ -122,9 +159,21 @@ module "project" {
   policy_list                = try(var.org_policies.policy_list, {})
   service_encryption_key_ids = var.kms_service_agents
   services                   = local.services
-  shared_vpc_service_config = {
-    attach       = local.vpc_setup
-    host_project = local.vpc_host_project
+  shared_vpc_service_config = var.vpc == null ? null : {
+    host_project = local.vpc.host_project
+    # these are non-authoritative
+    service_identity_iam = {
+      "roles/compute.networkUser" = compact([
+        local.vpc_gke_service_agent ? "container-engine" : null,
+        local.vpc_cloudservices ? "cloudservices" : null
+      ])
+      "roles/compute.securityAdmin" = compact([
+        local.vpc_gke_security_admin ? "container-engine" : null,
+      ])
+      "roles/container.hostServiceAgentUser" = compact([
+        local.vpc_gke_service_agent ? "container-engine" : null
+      ])
+    }
   }
 }
 
@@ -135,19 +184,11 @@ module "service-accounts" {
   project_id = module.project.project_id
 }
 
-# TODO(jccb): we should probably change this to non-authoritative bindings
-resource "google_compute_subnetwork_iam_binding" "binding" {
-  for_each   = local.vpc_setup ? coalesce(var.vpc.subnets_iam, {}) : {}
-  project    = local.vpc_host_project
-  subnetwork = "projects/${local.vpc_host_project}/regions/${split("/", each.key)[0]}/subnetworks/${split("/", each.key)[1]}"
-  region     = split("/", each.key)[0]
+resource "google_compute_subnetwork_iam_member" "default" {
+  for_each   = local.vpc_subnet_bindings
+  project    = local.vpc.host_project
+  subnetwork = "projects/${local.vpc.host_project}/regions/${each.value.region}/subnetworks/${each.value.subnet}"
+  region     = each.value.region
   role       = "roles/compute.networkUser"
-  members    = concat(each.value, local.network_user_service_accounts)
-}
-
-resource "google_project_iam_member" "host_project_bindings" {
-  for_each = local.host_project_bindings
-  project  = local.vpc_host_project
-  role     = each.key
-  member   = each.value
+  member     = each.value.member
 }

examples/factories/project-factory/outputs.tf

@@ -21,8 +21,7 @@ output "project" {
   value       = module.project
 
   depends_on = [
-    google_compute_subnetwork_iam_binding.binding,
-    google_project_iam_member.host_project_bindings,
+    google_compute_subnetwork_iam_member.default,
     module.dns
   ]
 }
@@ -31,8 +30,7 @@ output "project_id" {
   description = "Project ID."
   value       = module.project.project_id
   depends_on = [
-    google_compute_subnetwork_iam_binding.binding,
-    google_project_iam_member.host_project_bindings,
+    google_compute_subnetwork_iam_member.default,
     module.dns
   ]
 }

examples/factories/project-factory/variables.tf

@@ -130,12 +130,14 @@ variable "services" {
   description = "Services to be enabled for the project."
   type        = list(string)
   default     = []
+  nullable    = false
 }
 
-variable "services_iam" {
-  description = "Custom IAM settings for robot ServiceAccounts in service => [role] format."
+variable "service_identities_iam" {
+  description = "Custom IAM settings for service identities in service => [role] format."
   type        = map(list(string))
   default     = {}
+  nullable    = false
 }
 
 variable "vpc" {

examples/networking/filtering-proxy/main.tf

@@ -252,8 +252,10 @@ module "project-app" {
   prefix          = var.prefix
   services        = ["compute.googleapis.com"]
   shared_vpc_service_config = {
-    attach       = true
     host_project = module.project-host.project_id
+    service_identity_iam = {
+      "roles/compute.networkUser" = ["cloudservices"]
+    }
   }
 }
 

examples/networking/shared-vpc-gke/main.tf

@@ -31,9 +31,6 @@ module "project-host" {
     service_projects = [] # defined later
   }
   iam = {
-    "roles/container.hostServiceAgentUser" = [
-      "serviceAccount:${module.project-svc-gke.service_accounts.robots.container-engine}"
-    ]
     "roles/owner" = var.owners_host
   }
 }
@@ -48,8 +45,10 @@ module "project-svc-gce" {
   oslogin         = true
   oslogin_admins  = var.owners_gce
   shared_vpc_service_config = {
-    attach       = true
     host_project = module.project-host.project_id
+    service_identity_iam = {
+      "roles/compute.networkUser" = ["cloudservices"]
+    }
   }
   iam = {
     "roles/owner" = var.owners_gce
@@ -67,8 +66,11 @@ module "project-svc-gke" {
   name            = "gke"
   services        = var.project_services
   shared_vpc_service_config = {
-    attach       = true
     host_project = module.project-host.project_id
+    service_identity_iam = {
+      "roles/container.hostServiceAgentUser" = ["container-engine"]
+      "roles/compute.networkUser"            = ["container-engine"]
+    }
   }
   iam = merge(
     {

fast/assets/schemas/project.schema.yaml

@@ -25,7 +25,7 @@ org_policies: include('org_policies', required=False)
 secrets: map(list(str()), key=str(), required=False)
 service_accounts: map(list(str()), required=False)
 services: list(str(matches='^[a-z-]*\.googleapis\.com$'), required=False)
-services_iam: map(list(str()), key=str(), required=False)
+service_identities_iam: map(list(str()), key=str(), required=False)
 vpc: include('vpc', required=False)
 ---
 billing_alert:

fast/stages/03-project-factory/prod/data/projects/project.yaml

@@ -72,8 +72,8 @@ services:
   - stackdriver.googleapis.com
   - compute.googleapis.com
 
-# [opt] Roles to assign to the robots service accounts in robot => [roles] format
-services_iam:
+# [opt] Roles to assign to the service identities in service => [roles] format
+service_identities_iam:
   compute:
     - roles/storage.objectViewer
 

modules/project/README.md

@@ -36,14 +36,46 @@ module "project" {
   name            = "project-example"
 
   iam_additive = {
-    "roles/viewer"               = ["group:one@example.org", "group:two@xample.org"],
-    "roles/storage.objectAdmin"  = ["group:two@example.org"],
-    "roles/owner"                = ["group:three@example.org"],
+    "roles/viewer"               = [
+      "group:one@example.org", "group:two@xample.org"
+    ],
+    "roles/storage.objectAdmin"  = [
+      "group:two@example.org"
+    ],
+    "roles/owner"                = [
+      "group:three@example.org"
+    ],
   }
 }
 # tftest modules=1 resources=5
 ```
 
+### Shared VPC service
+
+```hcl
+module "project" {
+  source          = "./modules/project"
+  name            = "project-example"
+
+  shared_vpc_service_config = {
+    attach               = true
+    host_project         = "my-host-project"
+    service_identity_iam = {
+      "roles/compute.networkUser"            = [
+        "cloudservices", "container-engine"
+      ]
+      "roles/vpcaccess.user"                 = [
+        "cloudrun"
+      ]
+      "roles/container.hostServiceAgentUser" = [
+        "container-engine"
+      ]
+    }
+  }
+}
+# tftest modules=1 resources=6
+```
+
 ### Organization policies
 
 ```hcl
@@ -74,6 +106,7 @@ module "project" {
 ```
 
 ## Logging Sinks
+
 ```hcl
 module "gcs" {
   source        = "./modules/gcs"
@@ -187,7 +220,7 @@ module "project" {
 | [organization-policies.tf](./organization-policies.tf) | Project-level organization policies. | <code>google_project_organization_policy</code> |
 | [outputs.tf](./outputs.tf) | Module outputs. |  |
 | [service-accounts.tf](./service-accounts.tf) | Service identities and supporting resources. | <code>google_kms_crypto_key_iam_member</code> · <code>google_project_service_identity</code> |
-| [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | <code>google_compute_shared_vpc_host_project</code> · <code>google_compute_shared_vpc_service_project</code> |
+| [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | <code>google_compute_shared_vpc_host_project</code> · <code>google_compute_shared_vpc_service_project</code> · <code>google_project_iam_member</code> |
 | [variables.tf](./variables.tf) | Module variables. |  |
 | [versions.tf](./versions.tf) | Version pins. |  |
 | [vpc-sc.tf](./vpc-sc.tf) | VPC-SC project-level perimeter configuration. | <code>google_access_context_manager_service_perimeter_resource</code> |
@@ -224,9 +257,9 @@ module "project" {
 | [service_perimeter_bridges](variables.tf#L211) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | <code>list&#40;string&#41;</code> |  | <code>null</code> |
 | [service_perimeter_standard](variables.tf#L218) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | <code>string</code> |  | <code>null</code> |
 | [services](variables.tf#L224) | Service APIs to enable. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
-| [shared_vpc_host_config](variables.tf#L230) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | <code title="object&#40;&#123;&#10;  enabled          &#61; bool&#10;  service_projects &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  enabled          &#61; false&#10;  service_projects &#61; &#91;&#93;&#10;&#125;">&#123;&#8230;&#125;</code> |
-| [shared_vpc_service_config](variables.tf#L243) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | <code title="object&#40;&#123;&#10;  attach       &#61; bool&#10;  host_project &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  attach       &#61; false&#10;  host_project &#61; &#34;&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
-| [skip_delete](variables.tf#L256) | Allows the underlying resources to be destroyed without destroying the project itself. | <code>bool</code> |  | <code>false</code> |
+| [shared_vpc_host_config](variables.tf#L230) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | <code title="object&#40;&#123;&#10;  enabled          &#61; bool&#10;  service_projects &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [shared_vpc_service_config](variables.tf#L239) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | <code title="object&#40;&#123;&#10;  host_project         &#61; string&#10;  service_identity_iam &#61; map&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [skip_delete](variables.tf#L249) | Allows the underlying resources to be destroyed without destroying the project itself. | <code>bool</code> |  | <code>false</code> |
 
 ## Outputs
 
@@ -234,9 +267,9 @@ module "project" {
 |---|---|:---:|
 | [custom_roles](outputs.tf#L17) | Ids of the created custom roles. |  |
 | [name](outputs.tf#L25) | Project name. |  |
-| [number](outputs.tf#L37) | Project number. |  |
-| [project_id](outputs.tf#L49) | Project id. |  |
-| [service_accounts](outputs.tf#L63) | Product robot service accounts in project. |  |
-| [sink_writer_identities](outputs.tf#L79) | Writer identities created for each sink. |  |
+| [number](outputs.tf#L38) | Project number. |  |
+| [project_id](outputs.tf#L51) | Project id. |  |
+| [service_accounts](outputs.tf#L66) | Product robot service accounts in project. |  |
+| [sink_writer_identities](outputs.tf#L82) | Writer identities created for each sink. |  |
 
 <!-- END TFDOC -->

modules/project/outputs.tf

@@ -30,6 +30,7 @@ output "name" {
     google_project_organization_policy.list,
     google_project_service.project_services,
     google_compute_shared_vpc_service_project.service_projects,
+    google_project_iam_member.shared_vpc_host_robots,
     google_kms_crypto_key_iam_member.service_identity_cmek
   ]
 }
@@ -42,6 +43,7 @@ output "number" {
     google_project_organization_policy.list,
     google_project_service.project_services,
     google_compute_shared_vpc_service_project.service_projects,
+    google_project_iam_member.shared_vpc_host_robots,
     google_kms_crypto_key_iam_member.service_identity_cmek
   ]
 }
@@ -56,6 +58,7 @@ output "project_id" {
     google_project_organization_policy.list,
     google_project_service.project_services,
     google_compute_shared_vpc_service_project.service_projects,
+    google_project_iam_member.shared_vpc_host_robots,
     google_kms_crypto_key_iam_member.service_identity_cmek
   ]
 }

modules/project/service-accounts.tf

@@ -29,6 +29,8 @@ locals {
     bq                = "bq-%s@bigquery-encryption"
     cloudasset        = "service-%s@gcp-sa-cloudasset"
     cloudbuild        = "service-%s@gcp-sa-cloudbuild"
+    cloudfunctions    = "service-%s@gcf-admin-robot"
+    cloudrun          = "service-%s@serverless-robot-prod"
     composer          = "service-%s@cloudcomposer-accounts"
     compute           = "service-%s@compute-system"
     container-engine  = "service-%s@container-engine-robot"
@@ -36,10 +38,11 @@ locals {
     dataflow          = "service-%s@dataflow-service-producer-prod"
     dataproc          = "service-%s@dataproc-accounts"
     gae-flex          = "service-%s@gae-api-prod"
-    gcf               = "service-%s@gcf-admin-robot"
-    pubsub            = "service-%s@gcp-sa-pubsub"
-    secretmanager     = "service-%s@gcp-sa-secretmanager"
-    storage           = "service-%s@gs-project-accounts"
+    # TODO: deprecate gcf
+    gcf           = "service-%s@gcf-admin-robot"
+    pubsub        = "service-%s@gcp-sa-pubsub"
+    secretmanager = "service-%s@gcp-sa-secretmanager"
+    storage       = "service-%s@gs-project-accounts"
   }
   service_accounts_default = {
     compute = "${local.project.number}-compute@developer.gserviceaccount.com"

modules/project/shared-vpc.tf

@@ -16,19 +16,41 @@
 
 # tfdoc:file:description Shared VPC project-level configuration.
 
+locals {
+  # compute the host project IAM bindings for this project's service identities
+  _svpc_service_identity_iam = coalesce(
+    local.svpc_service_config.service_identity_iam, {}
+  )
+  _svpc_service_iam = flatten([
+    for role, services in local._svpc_service_identity_iam : [
+      for service in services : { role = role, service = service }
+    ]
+  ])
+  svpc_host_config = {
+    enabled = coalesce(
+      try(var.shared_vpc_host_config.enabled, null), false
+    )
+    service_projects = coalesce(
+      try(var.shared_vpc_host_config.service_projects, null), []
+    )
+  }
+  svpc_service_config = coalesce(var.shared_vpc_service_config, {
+    host_project = null, service_identity_iam = {}
+  })
+  svpc_service_iam = {
+    for b in local._svpc_service_iam : "${b.role}:${b.service}" => b
+  }
+}
+
 resource "google_compute_shared_vpc_host_project" "shared_vpc_host" {
   provider = google-beta
-  count    = try(var.shared_vpc_host_config.enabled, false) ? 1 : 0
+  count    = local.svpc_host_config.enabled ? 1 : 0
   project  = local.project.project_id
 }
 
 resource "google_compute_shared_vpc_service_project" "service_projects" {
-  provider = google-beta
-  for_each = (
-    try(var.shared_vpc_host_config.enabled, false)
-    ? toset(coalesce(var.shared_vpc_host_config.service_projects, []))
-    : toset([])
-  )
+  provider        = google-beta
+  for_each        = toset(local.svpc_host_config.service_projects)
   host_project    = local.project.project_id
   service_project = each.value
   depends_on      = [google_compute_shared_vpc_host_project.shared_vpc_host]
@@ -36,7 +58,19 @@ resource "google_compute_shared_vpc_service_project" "service_projects" {
 
 resource "google_compute_shared_vpc_service_project" "shared_vpc_service" {
   provider        = google-beta
-  count           = try(var.shared_vpc_service_config.attach, false) ? 1 : 0
+  count           = local.svpc_service_config.host_project != null ? 1 : 0
   host_project    = var.shared_vpc_service_config.host_project
   service_project = local.project.project_id
 }
+
+resource "google_project_iam_member" "shared_vpc_host_robots" {
+  for_each = local.svpc_service_iam
+  project  = var.shared_vpc_service_config.host_project
+  role     = each.value.role
+  member = (
+    each.value.service == "cloudservices"
+    ? local.service_account_cloud_services
+    : local.service_accounts_robots[each.value.service]
+  )
+}
+

modules/project/variables.tf

@@ -233,24 +233,17 @@ variable "shared_vpc_host_config" {
     enabled          = bool
     service_projects = list(string)
   })
-  default = {
-    enabled          = false
-    service_projects = []
-  }
-  nullable = false
+  default = null
 }
 
 variable "shared_vpc_service_config" {
   description = "Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config)."
+  # the list of valid service identities is in service-accounts.tf
   type = object({
-    attach       = bool
-    host_project = string
+    host_project         = string
+    service_identity_iam = map(list(string))
   })
-  default = {
-    attach       = false
-    host_project = ""
-  }
-  nullable = false
+  default = null
 }
 
 variable "skip_delete" {

tests/examples/factories/project_factory/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/examples/factories/project_factory/fixture/defaults.yaml

@@ -0,0 +1,24 @@
+# skip boilerplate check
+
+billing_account_id: 012345-67890A-BCDEF0
+
+# [opt] Setup for billing alerts
+billing_alert:
+  amount: 1000
+  thresholds:
+    current: [0.5, 0.8]
+    forecasted: [0.5, 0.8]
+  credit_treatment: INCLUDE_ALL_CREDITS
+
+# [opt] Contacts for billing alerts and important notifications
+essential_contacts: ["team-contacts@example.com"]
+
+# [opt] Labels set for all projects
+labels:
+  environment: prod
+  department: accounting
+  application: example-app
+  foo: bar
+
+# [opt] Additional notification channels for billing
+notification_channels: []

tests/examples/factories/project_factory/fixture/main.tf

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  _defaults = yamldecode(file(var.defaults_file))
+  _defaults_net = {
+    billing_account_id   = var.billing_account_id
+    environment_dns_zone = var.environment_dns_zone
+    shared_vpc_self_link = var.shared_vpc_self_link
+    vpc_host_project     = var.vpc_host_project
+  }
+  defaults = merge(local._defaults, local._defaults_net)
+  projects = {
+    for f in fileset("${var.data_dir}", "**/*.yaml") :
+    trimsuffix(f, ".yaml") => yamldecode(file("${var.data_dir}/${f}"))
+  }
+}
+
+module "projects" {
+  source                 = "../../../../../examples/factories/project-factory"
+  for_each               = local.projects
+  defaults               = local.defaults
+  project_id             = each.key
+  billing_account_id     = try(each.value.billing_account_id, null)
+  billing_alert          = try(each.value.billing_alert, null)
+  dns_zones              = try(each.value.dns_zones, [])
+  essential_contacts     = try(each.value.essential_contacts, [])
+  folder_id              = each.value.folder_id
+  group_iam              = try(each.value.group_iam, {})
+  iam                    = try(each.value.iam, {})
+  kms_service_agents     = try(each.value.kms, {})
+  labels                 = try(each.value.labels, {})
+  org_policies           = try(each.value.org_policies, null)
+  service_accounts       = try(each.value.service_accounts, {})
+  services               = try(each.value.services, [])
+  service_identities_iam = try(each.value.service_identities_iam, {})
+  vpc                    = try(each.value.vpc, null)
+}

tests/examples/factories/project_factory/fixture/projects/project.yaml

@@ -0,0 +1,100 @@
+# skip boilerplate check
+
+# [opt] Billing account id - overrides default if set
+billing_account_id: 012345-67890A-BCDEF0
+
+# [opt] Billing alerts config - overrides default if set
+billing_alert:
+  amount: 10
+  thresholds:
+    current:
+      - 0.5
+      - 0.8
+    forecasted: []
+  credit_treatment: INCLUDE_ALL_CREDITS
+
+# [opt] DNS zones to be created as children of the environment_dns_zone defined in defaults
+dns_zones:
+  - lorem
+  - ipsum
+
+# [opt] Contacts for billing alerts and important notifications
+essential_contacts:
+  - team-a-contacts@example.com
+
+# Folder the project will be created as children of
+folder_id: folders/012345678901
+
+# [opt] Authoritative IAM bindings in group => [roles] format
+group_iam:
+  test-team-foobar@fast-lab-0.gcp-pso-italy.net:
+    - roles/compute.admin
+
+# [opt] Authoritative IAM bindings in role => [principals] format
+# Generally used to grant roles to service accounts external to the project
+iam:
+  roles/compute.admin:
+    - serviceAccount:service-account
+
+# [opt] Service robots and keys they will be assigned as cryptoKeyEncrypterDecrypter
+# in service => [keys] format
+kms_service_agents:
+  compute: [key1, key2]
+  storage: [key1, key2]
+
+# [opt] Labels for the project - merged with the ones defined in defaults
+labels:
+  environment: prod
+
+# [opt] Org policy overrides defined at project level
+org_policies:
+  policy_boolean:
+    constraints/compute.disableGuestAttributesAccess: true
+  policy_list:
+    constraints/compute.trustedImageProjects:
+      inherit_from_parent: null
+      status: true
+      suggested_value: null
+      values:
+        - projects/fast-prod-iac-core-0
+
+# [opt] Service account to create for the project and their roles on the project
+# in name => [roles] format
+service_accounts:
+  another-service-account:
+    - roles/compute.admin
+  my-service-account:
+    - roles/compute.admin
+
+# [opt] APIs to enable on the project.
+services:
+  - storage.googleapis.com
+  - stackdriver.googleapis.com
+  - compute.googleapis.com
+
+# [opt] Roles to assign to the service identities in service => [roles] format
+service_identities_iam:
+  compute:
+    - roles/storage.objectViewer
+
+  # [opt] VPC setup.
+  # If set enables the `compute.googleapis.com` service and configures
+  # service project attachment
+vpc:
+  # [opt] If set, enables the container API
+  gke_setup:
+    # Grants "roles/container.hostServiceAgentUser" to the container robot if set
+    enable_host_service_agent: false
+
+    # Grants  "roles/compute.securityAdmin" to the container robot if set
+    enable_security_admin: true
+
+  # Host project the project will be service project of
+  host_project: fast-prod-net-spoke-0
+
+  # [opt] Subnets in the host project where principals will be granted networkUser
+  # in region/subnet-name => [principals]
+  subnets_iam:
+    europe-west1/prod-default-ew1:
+      - user:foobar@example.com
+      - serviceAccount:service-account1

tests/examples/factories/project_factory/fixture/variables.tf

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "billing_account_id" {
+  description = "Billing account id."
+  type        = string
+  default     = "012345-67890A-BCDEF0"
+}
+
+variable "data_dir" {
+  description = "Relative path for the folder storing configuration data."
+  type        = string
+  default     = "./projects/"
+}
+
+variable "environment_dns_zone" {
+  description = "DNS zone suffix for environment."
+  type        = string
+  default     = "prod.gcp.example.com"
+}
+
+variable "defaults_file" {
+  description = "Relative path for the file storing the project factory configuration."
+  type        = string
+  default     = "./defaults.yaml"
+}
+
+variable "shared_vpc_self_link" {
+  description = "Self link for the shared VPC."
+  type        = string
+  default     = "self-link"
+}
+
+variable "vpc_host_project" {
+  # tfdoc:variable:source 02-networking
+  description = "Host project for the shared VPC."
+  type        = string
+  default     = "host-project"
+}

tests/examples/factories/project_factory/test_plan.py

@@ -0,0 +1,18 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def test_counts(e2e_plan_runner):
+  "Check for a clean plan"
+  modules, resources = e2e_plan_runner()
+  assert len(modules) > 0 and len(resources) > 0

tests/examples/networking/filtering_proxy/test_plan.py

@@ -16,8 +16,8 @@ def test_resources(e2e_plan_runner):
   "Test that plan works and the numbers of resources is as expected."
   modules, resources = e2e_plan_runner()
   assert len(modules) == 11
-  assert len(resources) == 29
+  assert len(resources) == 30
 
   modules, resources = e2e_plan_runner(mig="true")
   assert len(modules) == 13
-  assert len(resources) == 35
+  assert len(resources) == 36

tests/examples/networking/shared_vpc_gke/test_plan.py

@@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
   "Test that plan works and the numbers of resources is as expected."
   modules, resources = e2e_plan_runner()
   assert len(modules) == 10
-  assert len(resources) == 41
+  assert len(resources) == 43