diff --git a/blueprints/data-solutions/data-playground/README.md b/blueprints/data-solutions/data-playground/README.md
index ecc12973..db7d51fc 100644
--- a/blueprints/data-solutions/data-playground/README.md
+++ b/blueprints/data-solutions/data-playground/README.md
@@ -12,12 +12,15 @@ This sample creates several distinct groups of resources:
- project
- networking
- - VPC network with a default subnet and CloudNat
- - Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC
- Vertex AI Workbench notebook configured with a private IP and using a dedicated Service Account
- One GCS bucket
- One BigQuery dataset
+## Virtual Private Cloud (VPC) design
+As is often the case in real-world configurations, this blueprint accepts as input an existing Shared-VPC via the network_config variable. Make sure that 'container.googleapis.com', 'notebooks.googleapis.com' and 'servicenetworking.googleapis.com' are enabled in the VPC host project.
+
+If the network_config variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
+
## Deploy your enviroment
We assume the identiy running the following steps has the following role:
@@ -47,12 +50,12 @@ You can now connect to the Vertex AI notbook to perform your data analysy.
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [prefix](variables.tf#L22) | Prefix used for resource names. | string
| ✓ | |
-| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | string
| ✓ | |
+| [prefix](variables.tf#L32) | Prefix used for resource names. | string
| ✓ | |
+| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | string
| ✓ | |
| [location](variables.tf#L16) | The location where resources will be deployed. | string
| | "EU"
|
-| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…})
| | null
|
-| [region](variables.tf#L45) | The region where resources will be deployed. | string
| | "europe-west1"
|
-| [vpc_config](variables.tf#L61) | Parameters to create a VPC. | object({…})
| | {…}
|
+| [network_config](variables.tf#L22) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…})
| | null
|
+| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…})
| | null
|
+| [region](variables.tf#L55) | The region where resources will be deployed. | string
| | "europe-west1"
|
## Outputs
diff --git a/blueprints/data-solutions/data-playground/main.tf b/blueprints/data-solutions/data-playground/main.tf
index ff079d5e..b87e8e73 100644
--- a/blueprints/data-solutions/data-playground/main.tf
+++ b/blueprints/data-solutions/data-playground/main.tf
@@ -17,6 +17,39 @@
###############################################################################
locals {
service_encryption_keys = var.service_encryption_keys
+ shared_vpc_project = try(var.network_config.host_project, null)
+
+ subnet = (
+ local.use_shared_vpc
+ ? var.network_config.subnet_self_link
+ : values(module.vpc.0.subnet_self_links)[0]
+ )
+ vpc = (
+ local.use_shared_vpc
+ ? var.network_config.network_self_link
+ : module.vpc.0.self_link
+ )
+ use_shared_vpc = var.network_config != null
+
+ shared_vpc_bindings = {
+ "roles/compute.networkUser" = [
+ "robot-df", "notebooks"
+ ]
+ }
+
+ shared_vpc_role_members = {
+ robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}"
+ notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
+ }
+
+ # reassemble in a format suitable for for_each
+ shared_vpc_bindings_map = {
+ for binding in flatten([
+ for role, members in local.shared_vpc_bindings : [
+ for member in members : { role = role, member = member }
+ ]
+ ]) : "${binding.role}-${binding.member}" => binding
+ }
}
module "project" {
@@ -27,6 +60,7 @@ module "project" {
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
services = [
+ "aiplatform.googleapis.com",
"bigquery.googleapis.com",
"bigquerystorage.googleapis.com",
"bigqueryreservation.googleapis.com",
@@ -42,17 +76,26 @@ module "project" {
"storage.googleapis.com",
"storage-component.googleapis.com"
]
+
+ shared_vpc_service_config = local.shared_vpc_project == null ? null : {
+ attach = true
+ host_project = local.shared_vpc_project
+ }
+
org_policies = {
# "constraints/compute.requireOsLogin" = {
# enforce = false
# }
- # Example of applying a project wide policy, mainly useful for Composer
+ # Example of applying a project wide policy, mainly useful for Composer 1
}
service_encryption_key_ids = {
compute = [try(local.service_encryption_keys.compute, null)]
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
}
+ service_config = {
+ disable_on_destroy = false, disable_dependent_services = false
+ }
}
###############################################################################
@@ -61,11 +104,12 @@ module "project" {
module "vpc" {
source = "../../../modules/net-vpc"
+ count = local.use_shared_vpc ? 0 : 1
project_id = module.project.project_id
name = "${var.prefix}-vpc"
subnets = [
{
- ip_cidr_range = var.vpc_config.ip_cidr_range
+ ip_cidr_range = "10.0.0.0/20"
name = "${var.prefix}-subnet"
region = var.region
}
@@ -74,10 +118,11 @@ module "vpc" {
module "vpc-firewall" {
source = "../../../modules/net-vpc-firewall"
+ count = local.use_shared_vpc ? 0 : 1
project_id = module.project.project_id
- network = module.vpc.name
+ network = module.vpc.0.name
default_rules_config = {
- admin_ranges = [var.vpc_config.ip_cidr_range]
+ admin_ranges = ["10.0.0.0/20"]
}
ingress_rules = {
#TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed
@@ -92,12 +137,21 @@ module "vpc-firewall" {
module "cloudnat" {
source = "../../../modules/net-cloudnat"
+ count = local.use_shared_vpc ? 0 : 1
project_id = module.project.project_id
name = "${var.prefix}-default"
region = var.region
- router_network = module.vpc.name
+ router_network = module.vpc.0.name
}
+resource "google_project_iam_member" "shared_vpc" {
+ count = local.use_shared_vpc ? 1 : 0
+ project = var.network_config.host_project
+ role = "roles/compute.networkUser"
+ member = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
+}
+
+
###############################################################################
# Storage #
###############################################################################
@@ -121,8 +175,6 @@ module "dataset" {
###############################################################################
# Vertex AI Notebook #
###############################################################################
-# TODO: Add encryption_key to Vertex AI notebooks as well
-# TODO: Add shared VPC support
module "service-account-notebook" {
source = "../../../modules/iam-service-account"
@@ -160,11 +212,14 @@ resource "google_notebooks_instance" "playground" {
no_public_ip = true
no_proxy_access = false
- network = module.vpc.network.id
- subnet = module.vpc.subnets[format("%s/%s", var.region, "${var.prefix}-subnet")].id
+ network = local.vpc
+ subnet = local.subnet
service_account = module.service-account-notebook.email
#TODO Uncomment once terraform-provider-google/issues/9273 is fixed
# tags = ["ssh"]
+ depends_on = [
+ google_project_iam_member.shared_vpc,
+ ]
}
diff --git a/blueprints/data-solutions/data-playground/outputs.tf b/blueprints/data-solutions/data-playground/outputs.tf
index 4b80c311..35f2efeb 100644
--- a/blueprints/data-solutions/data-playground/outputs.tf
+++ b/blueprints/data-solutions/data-playground/outputs.tf
@@ -37,5 +37,5 @@ output "project" {
output "vpc" {
description = "VPC Network."
- value = module.vpc.name
+ value = local.vpc
}
diff --git a/blueprints/data-solutions/data-playground/variables.tf b/blueprints/data-solutions/data-playground/variables.tf
index 17354067..3bd0ca65 100644
--- a/blueprints/data-solutions/data-playground/variables.tf
+++ b/blueprints/data-solutions/data-playground/variables.tf
@@ -19,6 +19,16 @@ variable "location" {
default = "EU"
}
+variable "network_config" {
+ description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
+ type = object({
+ host_project = string
+ network_self_link = string
+ subnet_self_link = string
+ })
+ default = null
+}
+
variable "prefix" {
description = "Prefix used for resource names."
type = string
@@ -57,13 +67,3 @@ variable "service_encryption_keys" { # service encription key
})
default = null
}
-
-variable "vpc_config" {
- description = "Parameters to create a VPC."
- type = object({
- ip_cidr_range = string
- })
- default = {
- ip_cidr_range = "10.0.0.0/20"
- }
-}
diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf
index 9b1a6760..e07df577 100644
--- a/fast/stages/01-resman/outputs.tf
+++ b/fast/stages/01-resman/outputs.tf
@@ -76,11 +76,11 @@ locals {
data-platform-prod = try(module.branch-dp-prod-folder.0.id, null)
gke-dev = try(module.branch-gke-dev-folder.0.id, null)
gke-prod = try(module.branch-gke-prod-folder.0.id, null)
- networking = module.branch-network-folder.id
- networking-dev = module.branch-network-dev-folder.id
- networking-prod = module.branch-network-prod-folder.id
+ networking = try(module.branch-network-folder.id, null)
+ networking-dev = try(module.branch-network-dev-folder.id, null)
+ networking-prod = try(module.branch-network-prod-folder.id, null)
sandbox = try(module.branch-sandbox-folder.0.id, null)
- security = module.branch-security-folder.id
+ security = try(module.branch-security-folder.id, null)
teams = try(module.branch-teams-folder.0.id, null)
},
{
diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf
index e1f6cb71..abf34cae 100644
--- a/modules/project/service-accounts.tf
+++ b/modules/project/service-accounts.tf
@@ -45,6 +45,7 @@ locals {
# TODO: jit?
gke-mcs = "service-%s@gcp-sa-mcsd"
monitoring-notifications = "service-%s@gcp-sa-monitoring-notification"
+ notebooks = "service-%s@gcp-sa-notebooks"
pubsub = "service-%s@gcp-sa-pubsub"
secretmanager = "service-%s@gcp-sa-secretmanager"
sql = "service-%s@gcp-sa-cloud-sql"
diff --git a/tests/blueprints/data_solutions/data_playground/test_plan.py b/tests/blueprints/data_solutions/data_playground/test_plan.py
index a0c3b5e6..daaa57fc 100644
--- a/tests/blueprints/data_solutions/data_playground/test_plan.py
+++ b/tests/blueprints/data_solutions/data_playground/test_plan.py
@@ -22,4 +22,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner(FIXTURES_DIR)
assert len(modules) == 7
- assert len(resources) == 37
+ assert len(resources) == 38