Merge pull request #601 from GoogleCloudPlatform/lcaggio/gcs2bq_shared_vpc

[data-solutions/gcs-to-bq-with-least-privileges] Add support for shared VPC

examples/data-solutions/gcs-to-bq-with-least-privileges/README.md

@@ -74,6 +74,16 @@ $ terraform apply
 
 You should see the output of the Terraform script with resources created and some command pre-created for you to run the example following steps below.
 
+### Virtual Private Cloud (VPC) design
+
+As is often the case in real-world configurations, this example accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable.
+
+If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
+
+When `network_config` variable is configured, the identity running the Terraform script need to have the following roles:
+  - `roles/compute.xpnAdmin` on the host project folder or org
+  - `roles/resourcemanager.projectIamAdmin` on the host project, either with no conditions or with a condition allowing [delegated role grants](https://medium.com/google-cloud/managing-gcp-service-usage-through-delegated-role-grants-a843610f2226#:~:text=Delegated%20role%20grants%20is%20a,setIamPolicy%20permission%20on%20a%20resource.) for `roles/compute.networkUser`, `roles/composer.sharedVpcAgent`, `roles/container.hostServiceAgentUser`
+
 ## Test your environment with Cloud Dataflow
 
 We assume all those steps are run using a user listed on `data_eng_principals`. You can authenticate as the user using the following command:
@@ -131,13 +141,14 @@ bq query --use_legacy_sql=false 'SELECT * FROM `PROJECT.datalake.person` LIMIT 1
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [prefix](variables.tf#L26) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | <code>string</code> | ✓ |  |
+| [prefix](variables.tf#L36) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | <code>string</code> | ✓ |  |
-| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ |  |
+| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ |  |
 | [cmek_encryption](variables.tf#L15) | Flag to enable CMEK on GCP resources created. | <code>bool</code> |  | <code>false</code> |
 | [data_eng_principals](variables.tf#L21) | Groups with Service Account Token creator role on service accounts in IAM format, eg 'group:group@domain.com'. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
-| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; string&#10;  parent             &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [network_config](variables.tf#L27) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10;  host_project     &#61; string&#10;  subnet_self_link &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [region](variables.tf#L45) | The region where resources will be deployed. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
+| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; string&#10;  parent             &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [vpc_subnet_range](variables.tf#L51) | Ip range used for the VPC subnet created for the example. | <code>string</code> |  | <code>&#34;10.0.0.0&#47;20&#34;</code> |
+| [region](variables.tf#L55) | The region where resources will be deployed. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
+| [vpc_subnet_range](variables.tf#L61) | Ip range used for the VPC subnet created for the example. | <code>string</code> |  | <code>&#34;10.0.0.0&#47;20&#34;</code> |
 
 ## Outputs
 

examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf

@@ -76,6 +76,29 @@ locals {
       "serviceAccount:${module.project.service_accounts.robots.dataflow}"
     ]
   }
+  network_subnet_selflink = try(
+    module.vpc[0].subnets["${var.region}/subnet"].self_link,
+    var.network_config.subnet_self_link
+  )
+  shared_vpc_bindings = {
+    "roles/compute.networkUser" = [
+      "robot-df", "sa-df-worker"
+    ]
+  }
+  # reassemble in a format suitable for for_each
+  shared_vpc_bindings_map = {
+    for binding in flatten([
+      for role, members in local.shared_vpc_bindings : [
+        for member in members : { role = role, member = member }
+      ]
+    ]) : "${binding.role}-${binding.member}" => binding
+  }
+  shared_vpc_project = try(var.network_config.host_project, null)
+  shared_vpc_role_members = {
+    robot-df     = "serviceAccount:${module.project.service_accounts.robots.dataflow}"
+    sa-df-worker = module.service-account-df.iam_email
+  }
+  use_shared_vpc = var.network_config != null
 }
 
 module "project" {
@@ -100,7 +123,19 @@ module "project" {
   # additive IAM bindings avoid disrupting bindings in existing project
   iam          = var.project_create != null ? local.iam : {}
   iam_additive = var.project_create == null ? local.iam : {}
+  shared_vpc_service_config = local.shared_vpc_project == null ? null : {
+    attach               = true
+    host_project         = local.shared_vpc_project
+    service_identity_iam = {}
+  }
   service_config = {
     disable_on_destroy = false, disable_dependent_services = false
   }
 }
+
+resource "google_project_iam_member" "shared_vpc" {
+  for_each = local.use_shared_vpc ? local.shared_vpc_bindings_map : {}
+  project  = var.network_config.host_project
+  role     = each.value.role
+  member   = lookup(local.shared_vpc_role_members, each.value.member)
+}

examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf

@@ -51,7 +51,7 @@ output "command_02_dataflow" {
     sa_orch_email    = module.service-account-orch.email
     project_id       = module.project.project_id
     region           = var.region
-    subnet           = module.vpc.subnets["${var.region}/subnet"].self_link
+    subnet           = local.network_subnet_selflink
     gcs_df_stg       = format("%s/%s", module.gcs-df-tmp.url, "stg")
     sa_df_email      = module.service-account-df.email
     cmek_encryption  = var.cmek_encryption

examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf

@@ -23,6 +23,16 @@ variable "data_eng_principals" {
   type        = list(string)
   default     = []
 }
+
+variable "network_config" {
+  description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
+  type = object({
+    host_project     = string
+    subnet_self_link = string
+  })
+  default = null
+}
+
 variable "prefix" {
   description = "Unique prefix used for resource names. Not used for project if 'project_create' is null."
   type        = string

examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf

@@ -14,6 +14,7 @@
 
 module "vpc" {
   source     = "../../../modules/net-vpc"
+  count      = local.use_shared_vpc ? 0 : 1
   project_id = module.project.project_id
   name       = "${var.prefix}-vpc"
   subnets = [
@@ -28,15 +29,17 @@ module "vpc" {
 
 module "vpc-firewall" {
   source       = "../../../modules/net-vpc-firewall"
+  count        = local.use_shared_vpc ? 0 : 1
   project_id   = module.project.project_id
-  network      = module.vpc.name
+  network      = module.vpc[0].name
   admin_ranges = [var.vpc_subnet_range]
 }
 
 module "nat" {
   source         = "../../../modules/net-cloudnat"
+  count          = local.use_shared_vpc ? 0 : 1
   project_id     = module.project.project_id
   region         = var.region
   name           = "${var.prefix}-default"
-  router_network = module.vpc.name
+  router_network = module.vpc[0].name
 }