Add shared-vpc support on Data Playgroud blueprint
This commit is contained in:
lcaggio
parent
8c826c8c0a
commit
07a7be29e3
|
|
@ -12,12 +12,15 @@ This sample creates several distinct groups of resources:
|
|||
|
||||
- project
|
||||
- networking
|
||||
- VPC network with a default subnet and CloudNat
|
||||
- Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC
|
||||
- Vertex AI Workbench notebook configured with a private IP and using a dedicated Service Account
|
||||
- One GCS bucket
|
||||
- One BigQuery dataset
|
||||
|
||||
## Virtual Private Cloud (VPC) design
|
||||
As is often the case in real-world configurations, this blueprint accepts as input an existing Shared-VPC via the network_config variable. Make sure that 'container.googleapis.com', 'notebooks.googleapis.com' and 'servicenetworking.googleapis.com' are enabled in the VPC host project.
|
||||
|
||||
If the network_config variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
|
||||
|
||||
## Deploy your enviroment
|
||||
We assume the identiy running the following steps has the following role:
|
||||
|
||||
|
|
@ -47,12 +50,12 @@ You can now connect to the Vertex AI notbook to perform your data analysy.
|
|||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [prefix](variables.tf#L22) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ | |
|
||||
| [prefix](variables.tf#L32) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ | |
|
||||
| [location](variables.tf#L16) | The location where resources will be deployed. | <code>string</code> | | <code>"EU"</code> |
|
||||
| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L45) | The region where resources will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [vpc_config](variables.tf#L61) | Parameters to create a VPC. | <code title="object({ ip_cidr_range = string })">object({…})</code> | | <code title="{ ip_cidr_range = "10.0.0.0/20" }">{…}</code> |
|
||||
| [network_config](variables.tf#L22) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string network_self_link = string subnet_self_link = string })">object({…})</code> | | <code>null</code> |
|
||||
| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L55) | The region where resources will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,19 @@
|
|||
###############################################################################
|
||||
locals {
|
||||
service_encryption_keys = var.service_encryption_keys
|
||||
shared_vpc_project = try(var.network_config.host_project, null)
|
||||
|
||||
subnet = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.subnet_self_link
|
||||
: values(module.vpc.0.subnet_self_links)[0]
|
||||
)
|
||||
vpc = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.network_self_link
|
||||
: module.vpc.0.self_link
|
||||
)
|
||||
use_shared_vpc = var.network_config != null
|
||||
}
|
||||
|
||||
module "project" {
|
||||
|
|
@ -27,6 +40,7 @@ module "project" {
|
|||
project_create = var.project_create != null
|
||||
prefix = var.project_create == null ? null : var.prefix
|
||||
services = [
|
||||
"aiplatform.googleapis.com",
|
||||
"bigquery.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
|
|
@ -42,17 +56,26 @@ module "project" {
|
|||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
]
|
||||
|
||||
shared_vpc_service_config = local.shared_vpc_project == null ? null : {
|
||||
attach = true
|
||||
host_project = local.shared_vpc_project
|
||||
}
|
||||
|
||||
org_policies = {
|
||||
# "constraints/compute.requireOsLogin" = {
|
||||
# enforce = false
|
||||
# }
|
||||
# Example of applying a project wide policy, mainly useful for Composer
|
||||
# Example of applying a project wide policy, mainly useful for Composer 1
|
||||
}
|
||||
service_encryption_key_ids = {
|
||||
compute = [try(local.service_encryption_keys.compute, null)]
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
service_config = {
|
||||
disable_on_destroy = false, disable_dependent_services = false
|
||||
}
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
|
|
@ -61,11 +84,12 @@ module "project" {
|
|||
|
||||
module "vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
name = "${var.prefix}-vpc"
|
||||
subnets = [
|
||||
{
|
||||
ip_cidr_range = var.vpc_config.ip_cidr_range
|
||||
ip_cidr_range = "10.0.0.0/20"
|
||||
name = "${var.prefix}-subnet"
|
||||
region = var.region
|
||||
}
|
||||
|
|
@ -74,10 +98,11 @@ module "vpc" {
|
|||
|
||||
module "vpc-firewall" {
|
||||
source = "../../../modules/net-vpc-firewall"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
network = module.vpc.name
|
||||
network = module.vpc.0.name
|
||||
default_rules_config = {
|
||||
admin_ranges = [var.vpc_config.ip_cidr_range]
|
||||
admin_ranges = ["10.0.0.0/20"]
|
||||
}
|
||||
ingress_rules = {
|
||||
#TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed
|
||||
|
|
@ -92,12 +117,21 @@ module "vpc-firewall" {
|
|||
|
||||
module "cloudnat" {
|
||||
source = "../../../modules/net-cloudnat"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
region = var.region
|
||||
router_network = module.vpc.name
|
||||
router_network = module.vpc.0.name
|
||||
}
|
||||
|
||||
resource "google_project_iam_member" "shared_vpc" {
|
||||
count = local.use_shared_vpc ? 1 : 0
|
||||
project = var.network_config.host_project
|
||||
role = "roles/compute.networkUser"
|
||||
member = "serviceAccount:${module.project.service_accounts.robots.notebooks}"
|
||||
}
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Storage #
|
||||
###############################################################################
|
||||
|
|
@ -121,8 +155,6 @@ module "dataset" {
|
|||
###############################################################################
|
||||
# Vertex AI Notebook #
|
||||
###############################################################################
|
||||
# TODO: Add encryption_key to Vertex AI notebooks as well
|
||||
# TODO: Add shared VPC support
|
||||
|
||||
module "service-account-notebook" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
|
|
@ -160,11 +192,14 @@ resource "google_notebooks_instance" "playground" {
|
|||
no_public_ip = true
|
||||
no_proxy_access = false
|
||||
|
||||
network = module.vpc.network.id
|
||||
subnet = module.vpc.subnets[format("%s/%s", var.region, "${var.prefix}-subnet")].id
|
||||
network = local.vpc
|
||||
subnet = local.subnet
|
||||
|
||||
service_account = module.service-account-notebook.email
|
||||
|
||||
#TODO Uncomment once terraform-provider-google/issues/9273 is fixed
|
||||
# tags = ["ssh"]
|
||||
depends_on = [
|
||||
google_project_iam_member.shared_vpc,
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,5 +37,5 @@ output "project" {
|
|||
|
||||
output "vpc" {
|
||||
description = "VPC Network."
|
||||
value = module.vpc.name
|
||||
value = local.vpc
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,16 @@ variable "location" {
|
|||
default = "EU"
|
||||
}
|
||||
|
||||
variable "network_config" {
|
||||
description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
|
||||
type = object({
|
||||
host_project = string
|
||||
network_self_link = string
|
||||
subnet_self_link = string
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "prefix" {
|
||||
description = "Prefix used for resource names."
|
||||
type = string
|
||||
|
|
@ -57,13 +67,3 @@ variable "service_encryption_keys" { # service encription key
|
|||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "vpc_config" {
|
||||
description = "Parameters to create a VPC."
|
||||
type = object({
|
||||
ip_cidr_range = string
|
||||
})
|
||||
default = {
|
||||
ip_cidr_range = "10.0.0.0/20"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
module "test" {
|
||||
source = "./data-playground"
|
||||
prefix = "plg-202301"
|
||||
project_id = "cg1-dev-data-cmn-0"
|
||||
network_config = {
|
||||
host_project = "cgg-dev-net-spoke-0"
|
||||
network_self_link = "https://www.googleapis.com/compute/v1/projects/cgg-dev-net-spoke-0/global/networks/dev-spoke-0"
|
||||
subnet_self_link = "https://www.googleapis.com/compute/v1/projects/cgg-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-dataplatform-ew1"
|
||||
}
|
||||
}
|
||||
|
|
@ -45,6 +45,7 @@ locals {
|
|||
# TODO: jit?
|
||||
gke-mcs = "service-%s@gcp-sa-mcsd"
|
||||
monitoring-notifications = "service-%s@gcp-sa-monitoring-notification"
|
||||
notebooks = "service-%s@gcp-sa-notebooks"
|
||||
pubsub = "service-%s@gcp-sa-pubsub"
|
||||
secretmanager = "service-%s@gcp-sa-secretmanager"
|
||||
sql = "service-%s@gcp-sa-cloud-sql"
|
||||
|
|
|
|||
|
|
@ -22,4 +22,4 @@ def test_resources(e2e_plan_runner):
|
|||
"Test that plan works and the numbers of resources is as expected."
|
||||
modules, resources = e2e_plan_runner(FIXTURES_DIR)
|
||||
assert len(modules) == 7
|
||||
assert len(resources) == 37
|
||||
assert len(resources) == 38
|
||||
|
|
|
|||
Loading…
Reference in New Issue