Merge pull request #1098 from GoogleCloudPlatform/lcaggio/vertex-svpc
Add shared-vpc support on data-playground blueprint
This commit is contained in:
commit
1fe108b890
|
@ -12,12 +12,15 @@ This sample creates several distinct groups of resources:
|
||||||
|
|
||||||
- project
|
- project
|
||||||
- networking
|
- networking
|
||||||
- VPC network with a default subnet and CloudNat
|
|
||||||
- Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC
|
|
||||||
- Vertex AI Workbench notebook configured with a private IP and using a dedicated Service Account
|
- Vertex AI Workbench notebook configured with a private IP and using a dedicated Service Account
|
||||||
- One GCS bucket
|
- One GCS bucket
|
||||||
- One BigQuery dataset
|
- One BigQuery dataset
|
||||||
|
|
||||||
|
## Virtual Private Cloud (VPC) design
|
||||||
|
As is often the case in real-world configurations, this blueprint accepts as input an existing Shared-VPC via the network_config variable. Make sure that 'container.googleapis.com', 'notebooks.googleapis.com' and 'servicenetworking.googleapis.com' are enabled in the VPC host project.
|
||||||
|
|
||||||
|
If the network_config variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
|
||||||
|
|
||||||
## Deploy your enviroment
|
## Deploy your enviroment
|
||||||
We assume the identiy running the following steps has the following role:
|
We assume the identiy running the following steps has the following role:
|
||||||
|
|
||||||
|
@ -47,12 +50,12 @@ You can now connect to the Vertex AI notbook to perform your data analysy.
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [prefix](variables.tf#L22) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
| [prefix](variables.tf#L32) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||||
| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ | |
|
| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ | |
|
||||||
| [location](variables.tf#L16) | The location where resources will be deployed. | <code>string</code> | | <code>"EU"</code> |
|
| [location](variables.tf#L16) | The location where resources will be deployed. | <code>string</code> | | <code>"EU"</code> |
|
||||||
| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
| [network_config](variables.tf#L22) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string network_self_link = string subnet_self_link = string })">object({…})</code> | | <code>null</code> |
|
||||||
| [region](variables.tf#L45) | The region where resources will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
||||||
| [vpc_config](variables.tf#L61) | Parameters to create a VPC. | <code title="object({ ip_cidr_range = string })">object({…})</code> | | <code title="{ ip_cidr_range = "10.0.0.0/20" }">{…}</code> |
|
| [region](variables.tf#L55) | The region where resources will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
|
|
|
@ -17,6 +17,39 @@
|
||||||
###############################################################################
|
###############################################################################
|
||||||
locals {
|
locals {
|
||||||
service_encryption_keys = var.service_encryption_keys
|
service_encryption_keys = var.service_encryption_keys
|
||||||
|
shared_vpc_project = try(var.network_config.host_project, null)
|
||||||
|
|
||||||
|
subnet = (
|
||||||
|
local.use_shared_vpc
|
||||||
|
? var.network_config.subnet_self_link
|
||||||
|
: values(module.vpc.0.subnet_self_links)[0]
|
||||||
|
)
|
||||||
|
vpc = (
|
||||||
|
local.use_shared_vpc
|
||||||
|
? var.network_config.network_self_link
|
||||||
|
: module.vpc.0.self_link
|
||||||
|
)
|
||||||
|
use_shared_vpc = var.network_config != null
|
||||||
|
|
||||||
|
shared_vpc_bindings = {
|
||||||
|
"roles/compute.networkUser" = [
|
||||||
|
"robot-df", "notebooks"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
shared_vpc_role_members = {
|
||||||
|
robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}"
|
||||||
|
notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# reassemble in a format suitable for for_each
|
||||||
|
shared_vpc_bindings_map = {
|
||||||
|
for binding in flatten([
|
||||||
|
for role, members in local.shared_vpc_bindings : [
|
||||||
|
for member in members : { role = role, member = member }
|
||||||
|
]
|
||||||
|
]) : "${binding.role}-${binding.member}" => binding
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module "project" {
|
module "project" {
|
||||||
|
@ -27,6 +60,7 @@ module "project" {
|
||||||
project_create = var.project_create != null
|
project_create = var.project_create != null
|
||||||
prefix = var.project_create == null ? null : var.prefix
|
prefix = var.project_create == null ? null : var.prefix
|
||||||
services = [
|
services = [
|
||||||
|
"aiplatform.googleapis.com",
|
||||||
"bigquery.googleapis.com",
|
"bigquery.googleapis.com",
|
||||||
"bigquerystorage.googleapis.com",
|
"bigquerystorage.googleapis.com",
|
||||||
"bigqueryreservation.googleapis.com",
|
"bigqueryreservation.googleapis.com",
|
||||||
|
@ -42,17 +76,26 @@ module "project" {
|
||||||
"storage.googleapis.com",
|
"storage.googleapis.com",
|
||||||
"storage-component.googleapis.com"
|
"storage-component.googleapis.com"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
shared_vpc_service_config = local.shared_vpc_project == null ? null : {
|
||||||
|
attach = true
|
||||||
|
host_project = local.shared_vpc_project
|
||||||
|
}
|
||||||
|
|
||||||
org_policies = {
|
org_policies = {
|
||||||
# "constraints/compute.requireOsLogin" = {
|
# "constraints/compute.requireOsLogin" = {
|
||||||
# enforce = false
|
# enforce = false
|
||||||
# }
|
# }
|
||||||
# Example of applying a project wide policy, mainly useful for Composer
|
# Example of applying a project wide policy, mainly useful for Composer 1
|
||||||
}
|
}
|
||||||
service_encryption_key_ids = {
|
service_encryption_key_ids = {
|
||||||
compute = [try(local.service_encryption_keys.compute, null)]
|
compute = [try(local.service_encryption_keys.compute, null)]
|
||||||
bq = [try(local.service_encryption_keys.bq, null)]
|
bq = [try(local.service_encryption_keys.bq, null)]
|
||||||
storage = [try(local.service_encryption_keys.storage, null)]
|
storage = [try(local.service_encryption_keys.storage, null)]
|
||||||
}
|
}
|
||||||
|
service_config = {
|
||||||
|
disable_on_destroy = false, disable_dependent_services = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
|
@ -61,11 +104,12 @@ module "project" {
|
||||||
|
|
||||||
module "vpc" {
|
module "vpc" {
|
||||||
source = "../../../modules/net-vpc"
|
source = "../../../modules/net-vpc"
|
||||||
|
count = local.use_shared_vpc ? 0 : 1
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = "${var.prefix}-vpc"
|
name = "${var.prefix}-vpc"
|
||||||
subnets = [
|
subnets = [
|
||||||
{
|
{
|
||||||
ip_cidr_range = var.vpc_config.ip_cidr_range
|
ip_cidr_range = "10.0.0.0/20"
|
||||||
name = "${var.prefix}-subnet"
|
name = "${var.prefix}-subnet"
|
||||||
region = var.region
|
region = var.region
|
||||||
}
|
}
|
||||||
|
@ -74,10 +118,11 @@ module "vpc" {
|
||||||
|
|
||||||
module "vpc-firewall" {
|
module "vpc-firewall" {
|
||||||
source = "../../../modules/net-vpc-firewall"
|
source = "../../../modules/net-vpc-firewall"
|
||||||
|
count = local.use_shared_vpc ? 0 : 1
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
network = module.vpc.name
|
network = module.vpc.0.name
|
||||||
default_rules_config = {
|
default_rules_config = {
|
||||||
admin_ranges = [var.vpc_config.ip_cidr_range]
|
admin_ranges = ["10.0.0.0/20"]
|
||||||
}
|
}
|
||||||
ingress_rules = {
|
ingress_rules = {
|
||||||
#TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed
|
#TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed
|
||||||
|
@ -92,12 +137,21 @@ module "vpc-firewall" {
|
||||||
|
|
||||||
module "cloudnat" {
|
module "cloudnat" {
|
||||||
source = "../../../modules/net-cloudnat"
|
source = "../../../modules/net-cloudnat"
|
||||||
|
count = local.use_shared_vpc ? 0 : 1
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = "${var.prefix}-default"
|
name = "${var.prefix}-default"
|
||||||
region = var.region
|
region = var.region
|
||||||
router_network = module.vpc.name
|
router_network = module.vpc.0.name
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "google_project_iam_member" "shared_vpc" {
|
||||||
|
count = local.use_shared_vpc ? 1 : 0
|
||||||
|
project = var.network_config.host_project
|
||||||
|
role = "roles/compute.networkUser"
|
||||||
|
member = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# Storage #
|
# Storage #
|
||||||
###############################################################################
|
###############################################################################
|
||||||
|
@ -121,8 +175,6 @@ module "dataset" {
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# Vertex AI Notebook #
|
# Vertex AI Notebook #
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# TODO: Add encryption_key to Vertex AI notebooks as well
|
|
||||||
# TODO: Add shared VPC support
|
|
||||||
|
|
||||||
module "service-account-notebook" {
|
module "service-account-notebook" {
|
||||||
source = "../../../modules/iam-service-account"
|
source = "../../../modules/iam-service-account"
|
||||||
|
@ -160,11 +212,14 @@ resource "google_notebooks_instance" "playground" {
|
||||||
no_public_ip = true
|
no_public_ip = true
|
||||||
no_proxy_access = false
|
no_proxy_access = false
|
||||||
|
|
||||||
network = module.vpc.network.id
|
network = local.vpc
|
||||||
subnet = module.vpc.subnets[format("%s/%s", var.region, "${var.prefix}-subnet")].id
|
subnet = local.subnet
|
||||||
|
|
||||||
service_account = module.service-account-notebook.email
|
service_account = module.service-account-notebook.email
|
||||||
|
|
||||||
#TODO Uncomment once terraform-provider-google/issues/9273 is fixed
|
#TODO Uncomment once terraform-provider-google/issues/9273 is fixed
|
||||||
# tags = ["ssh"]
|
# tags = ["ssh"]
|
||||||
|
depends_on = [
|
||||||
|
google_project_iam_member.shared_vpc,
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,5 +37,5 @@ output "project" {
|
||||||
|
|
||||||
output "vpc" {
|
output "vpc" {
|
||||||
description = "VPC Network."
|
description = "VPC Network."
|
||||||
value = module.vpc.name
|
value = local.vpc
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,6 +19,16 @@ variable "location" {
|
||||||
default = "EU"
|
default = "EU"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "network_config" {
|
||||||
|
description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
|
||||||
|
type = object({
|
||||||
|
host_project = string
|
||||||
|
network_self_link = string
|
||||||
|
subnet_self_link = string
|
||||||
|
})
|
||||||
|
default = null
|
||||||
|
}
|
||||||
|
|
||||||
variable "prefix" {
|
variable "prefix" {
|
||||||
description = "Prefix used for resource names."
|
description = "Prefix used for resource names."
|
||||||
type = string
|
type = string
|
||||||
|
@ -57,13 +67,3 @@ variable "service_encryption_keys" { # service encription key
|
||||||
})
|
})
|
||||||
default = null
|
default = null
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "vpc_config" {
|
|
||||||
description = "Parameters to create a VPC."
|
|
||||||
type = object({
|
|
||||||
ip_cidr_range = string
|
|
||||||
})
|
|
||||||
default = {
|
|
||||||
ip_cidr_range = "10.0.0.0/20"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -45,6 +45,7 @@ locals {
|
||||||
# TODO: jit?
|
# TODO: jit?
|
||||||
gke-mcs = "service-%s@gcp-sa-mcsd"
|
gke-mcs = "service-%s@gcp-sa-mcsd"
|
||||||
monitoring-notifications = "service-%s@gcp-sa-monitoring-notification"
|
monitoring-notifications = "service-%s@gcp-sa-monitoring-notification"
|
||||||
|
notebooks = "service-%s@gcp-sa-notebooks"
|
||||||
pubsub = "service-%s@gcp-sa-pubsub"
|
pubsub = "service-%s@gcp-sa-pubsub"
|
||||||
secretmanager = "service-%s@gcp-sa-secretmanager"
|
secretmanager = "service-%s@gcp-sa-secretmanager"
|
||||||
sql = "service-%s@gcp-sa-cloud-sql"
|
sql = "service-%s@gcp-sa-cloud-sql"
|
||||||
|
|
|
@ -22,4 +22,4 @@ def test_resources(e2e_plan_runner):
|
||||||
"Test that plan works and the numbers of resources is as expected."
|
"Test that plan works and the numbers of resources is as expected."
|
||||||
modules, resources = e2e_plan_runner(FIXTURES_DIR)
|
modules, resources = e2e_plan_runner(FIXTURES_DIR)
|
||||||
assert len(modules) == 7
|
assert len(modules) == 7
|
||||||
assert len(resources) == 37
|
assert len(resources) == 38
|
||||||
|
|
Loading…
Reference in New Issue