diff --git a/blueprints/data-solutions/data-playground/README.md b/blueprints/data-solutions/data-playground/README.md index ecc12973..db7d51fc 100644 --- a/blueprints/data-solutions/data-playground/README.md +++ b/blueprints/data-solutions/data-playground/README.md @@ -12,12 +12,15 @@ This sample creates several distinct groups of resources: - project - networking - - VPC network with a default subnet and CloudNat - - Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC - Vertex AI Workbench notebook configured with a private IP and using a dedicated Service Account - One GCS bucket - One BigQuery dataset +## Virtual Private Cloud (VPC) design +As is often the case in real-world configurations, this blueprint accepts as input an existing Shared-VPC via the network_config variable. Make sure that 'container.googleapis.com', 'notebooks.googleapis.com' and 'servicenetworking.googleapis.com' are enabled in the VPC host project. + +If the network_config variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration). + ## Deploy your enviroment We assume the identiy running the following steps has the following role: @@ -47,12 +50,12 @@ You can now connect to the Vertex AI notbook to perform your data analysy. | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L22) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [prefix](variables.tf#L32) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | string | ✓ | | | [location](variables.tf#L16) | The location where resources will be deployed. | string | | "EU" | -| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | -| [region](variables.tf#L45) | The region where resources will be deployed. | string | | "europe-west1" | -| [vpc_config](variables.tf#L61) | Parameters to create a VPC. | object({…}) | | {…} | +| [network_config](variables.tf#L22) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | +| [region](variables.tf#L55) | The region where resources will be deployed. | string | | "europe-west1" | ## Outputs diff --git a/blueprints/data-solutions/data-playground/main.tf b/blueprints/data-solutions/data-playground/main.tf index ff079d5e..b87e8e73 100644 --- a/blueprints/data-solutions/data-playground/main.tf +++ b/blueprints/data-solutions/data-playground/main.tf @@ -17,6 +17,39 @@ ############################################################################### locals { service_encryption_keys = var.service_encryption_keys + shared_vpc_project = try(var.network_config.host_project, null) + + subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_link + : values(module.vpc.0.subnet_self_links)[0] + ) + vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.vpc.0.self_link + ) + use_shared_vpc = var.network_config != null + + shared_vpc_bindings = { + "roles/compute.networkUser" = [ + "robot-df", "notebooks" + ] + } + + shared_vpc_role_members = { + robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}" + notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}" + } + + # reassemble in a format suitable for for_each + shared_vpc_bindings_map = { + for binding in flatten([ + for role, members in local.shared_vpc_bindings : [ + for member in members : { role = role, member = member } + ] + ]) : "${binding.role}-${binding.member}" => binding + } } module "project" { @@ -27,6 +60,7 @@ module "project" { project_create = var.project_create != null prefix = var.project_create == null ? null : var.prefix services = [ + "aiplatform.googleapis.com", "bigquery.googleapis.com", "bigquerystorage.googleapis.com", "bigqueryreservation.googleapis.com", @@ -42,17 +76,26 @@ module "project" { "storage.googleapis.com", "storage-component.googleapis.com" ] + + shared_vpc_service_config = local.shared_vpc_project == null ? null : { + attach = true + host_project = local.shared_vpc_project + } + org_policies = { # "constraints/compute.requireOsLogin" = { # enforce = false # } - # Example of applying a project wide policy, mainly useful for Composer + # Example of applying a project wide policy, mainly useful for Composer 1 } service_encryption_key_ids = { compute = [try(local.service_encryption_keys.compute, null)] bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] } + service_config = { + disable_on_destroy = false, disable_dependent_services = false + } } ############################################################################### @@ -61,11 +104,12 @@ module "project" { module "vpc" { source = "../../../modules/net-vpc" + count = local.use_shared_vpc ? 0 : 1 project_id = module.project.project_id name = "${var.prefix}-vpc" subnets = [ { - ip_cidr_range = var.vpc_config.ip_cidr_range + ip_cidr_range = "10.0.0.0/20" name = "${var.prefix}-subnet" region = var.region } @@ -74,10 +118,11 @@ module "vpc" { module "vpc-firewall" { source = "../../../modules/net-vpc-firewall" + count = local.use_shared_vpc ? 0 : 1 project_id = module.project.project_id - network = module.vpc.name + network = module.vpc.0.name default_rules_config = { - admin_ranges = [var.vpc_config.ip_cidr_range] + admin_ranges = ["10.0.0.0/20"] } ingress_rules = { #TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed @@ -92,12 +137,21 @@ module "vpc-firewall" { module "cloudnat" { source = "../../../modules/net-cloudnat" + count = local.use_shared_vpc ? 0 : 1 project_id = module.project.project_id name = "${var.prefix}-default" region = var.region - router_network = module.vpc.name + router_network = module.vpc.0.name } +resource "google_project_iam_member" "shared_vpc" { + count = local.use_shared_vpc ? 1 : 0 + project = var.network_config.host_project + role = "roles/compute.networkUser" + member = "serviceAccount:${module.project.service_accounts.robots.notebooks}" +} + + ############################################################################### # Storage # ############################################################################### @@ -121,8 +175,6 @@ module "dataset" { ############################################################################### # Vertex AI Notebook # ############################################################################### -# TODO: Add encryption_key to Vertex AI notebooks as well -# TODO: Add shared VPC support module "service-account-notebook" { source = "../../../modules/iam-service-account" @@ -160,11 +212,14 @@ resource "google_notebooks_instance" "playground" { no_public_ip = true no_proxy_access = false - network = module.vpc.network.id - subnet = module.vpc.subnets[format("%s/%s", var.region, "${var.prefix}-subnet")].id + network = local.vpc + subnet = local.subnet service_account = module.service-account-notebook.email #TODO Uncomment once terraform-provider-google/issues/9273 is fixed # tags = ["ssh"] + depends_on = [ + google_project_iam_member.shared_vpc, + ] } diff --git a/blueprints/data-solutions/data-playground/outputs.tf b/blueprints/data-solutions/data-playground/outputs.tf index 4b80c311..35f2efeb 100644 --- a/blueprints/data-solutions/data-playground/outputs.tf +++ b/blueprints/data-solutions/data-playground/outputs.tf @@ -37,5 +37,5 @@ output "project" { output "vpc" { description = "VPC Network." - value = module.vpc.name + value = local.vpc } diff --git a/blueprints/data-solutions/data-playground/variables.tf b/blueprints/data-solutions/data-playground/variables.tf index 17354067..3bd0ca65 100644 --- a/blueprints/data-solutions/data-playground/variables.tf +++ b/blueprints/data-solutions/data-playground/variables.tf @@ -19,6 +19,16 @@ variable "location" { default = "EU" } +variable "network_config" { + description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." + type = object({ + host_project = string + network_self_link = string + subnet_self_link = string + }) + default = null +} + variable "prefix" { description = "Prefix used for resource names." type = string @@ -57,13 +67,3 @@ variable "service_encryption_keys" { # service encription key }) default = null } - -variable "vpc_config" { - description = "Parameters to create a VPC." - type = object({ - ip_cidr_range = string - }) - default = { - ip_cidr_range = "10.0.0.0/20" - } -} diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf index 9b1a6760..e07df577 100644 --- a/fast/stages/01-resman/outputs.tf +++ b/fast/stages/01-resman/outputs.tf @@ -76,11 +76,11 @@ locals { data-platform-prod = try(module.branch-dp-prod-folder.0.id, null) gke-dev = try(module.branch-gke-dev-folder.0.id, null) gke-prod = try(module.branch-gke-prod-folder.0.id, null) - networking = module.branch-network-folder.id - networking-dev = module.branch-network-dev-folder.id - networking-prod = module.branch-network-prod-folder.id + networking = try(module.branch-network-folder.id, null) + networking-dev = try(module.branch-network-dev-folder.id, null) + networking-prod = try(module.branch-network-prod-folder.id, null) sandbox = try(module.branch-sandbox-folder.0.id, null) - security = module.branch-security-folder.id + security = try(module.branch-security-folder.id, null) teams = try(module.branch-teams-folder.0.id, null) }, { diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf index e1f6cb71..abf34cae 100644 --- a/modules/project/service-accounts.tf +++ b/modules/project/service-accounts.tf @@ -45,6 +45,7 @@ locals { # TODO: jit? gke-mcs = "service-%s@gcp-sa-mcsd" monitoring-notifications = "service-%s@gcp-sa-monitoring-notification" + notebooks = "service-%s@gcp-sa-notebooks" pubsub = "service-%s@gcp-sa-pubsub" secretmanager = "service-%s@gcp-sa-secretmanager" sql = "service-%s@gcp-sa-cloud-sql" diff --git a/tests/blueprints/data_solutions/data_playground/test_plan.py b/tests/blueprints/data_solutions/data_playground/test_plan.py index a0c3b5e6..daaa57fc 100644 --- a/tests/blueprints/data_solutions/data_playground/test_plan.py +++ b/tests/blueprints/data_solutions/data_playground/test_plan.py @@ -22,4 +22,4 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 7 - assert len(resources) == 37 + assert len(resources) == 38