From 50856e6951763237be2133781acb4a7714bc8c72 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Thu, 23 Feb 2023 18:36:03 +0100 Subject: [PATCH] First commit --- blueprints/data-solutions/bq-ml/README.md | 6 + blueprints/data-solutions/bq-ml/main.tf | 247 +++++++++++++++++++ blueprints/data-solutions/bq-ml/outputs.tf | 52 ++++ blueprints/data-solutions/bq-ml/variables.tf | 69 ++++++ blueprints/data-solutions/bq-ml/versions.tf | 29 +++ 5 files changed, 403 insertions(+) create mode 100644 blueprints/data-solutions/bq-ml/README.md create mode 100644 blueprints/data-solutions/bq-ml/main.tf create mode 100644 blueprints/data-solutions/bq-ml/outputs.tf create mode 100644 blueprints/data-solutions/bq-ml/variables.tf create mode 100644 blueprints/data-solutions/bq-ml/versions.tf diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md new file mode 100644 index 00000000..42e4832c --- /dev/null +++ b/blueprints/data-solutions/bq-ml/README.md @@ -0,0 +1,6 @@ +# BQ ML and Vertex Pipeline + +This blueprint creates #TODO + + + diff --git a/blueprints/data-solutions/bq-ml/main.tf b/blueprints/data-solutions/bq-ml/main.tf new file mode 100644 index 00000000..2d7ab45b --- /dev/null +++ b/blueprints/data-solutions/bq-ml/main.tf @@ -0,0 +1,247 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### +# Project # +############################################################################### +locals { + service_encryption_keys = var.service_encryption_keys + shared_vpc_project = try(var.network_config.host_project, null) + + subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_link + : values(module.vpc.0.subnet_self_links)[0] + ) + vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.vpc.0.self_link + ) + use_shared_vpc = var.network_config != null + + shared_vpc_bindings = { + "roles/compute.networkUser" = [ + "robot-df", "notebooks" + ] + } + + shared_vpc_role_members = { + robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}" + notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}" + } + + # reassemble in a format suitable for for_each + shared_vpc_bindings_map = { + for binding in flatten([ + for role, members in local.shared_vpc_bindings : [ + for member in members : { role = role, member = member } + ] + ]) : "${binding.role}-${binding.member}" => binding + } +} + +module "project" { + source = "../../../modules/project" + name = var.project_id + parent = try(var.project_create.parent, null) + billing_account = try(var.project_create.billing_account_id, null) + project_create = var.project_create != null + prefix = var.project_create == null ? null : var.prefix + services = [ + "aiplatform.googleapis.com", + "bigquery.googleapis.com", + "bigquerystorage.googleapis.com", + "bigqueryreservation.googleapis.com", + "compute.googleapis.com", + "ml.googleapis.com", + "notebooks.googleapis.com", + "servicenetworking.googleapis.com", + "stackdriver.googleapis.com", + "storage.googleapis.com", + "storage-component.googleapis.com" + ] + + shared_vpc_service_config = local.shared_vpc_project == null ? null : { + attach = true + host_project = local.shared_vpc_project + } + + service_encryption_key_ids = { + compute = [try(local.service_encryption_keys.compute, null)] + bq = [try(local.service_encryption_keys.bq, null)] + storage = [try(local.service_encryption_keys.storage, null)] + } + service_config = { + disable_on_destroy = false, disable_dependent_services = false + } +} + +############################################################################### +# Networking # +############################################################################### + +module "vpc" { + source = "../../../modules/net-vpc" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "${var.prefix}-vpc" + subnets = [ + { + ip_cidr_range = "10.0.0.0/20" + name = "${var.prefix}-subnet" + region = var.region + } + ] +} + +module "vpc-firewall" { + source = "../../../modules/net-vpc-firewall" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + network = module.vpc.0.name + default_rules_config = { + admin_ranges = ["10.0.0.0/20"] + } + ingress_rules = { + #TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed + ("${var.prefix}-iap") = { + description = "Enable SSH from IAP on Notebooks." + source_ranges = ["35.235.240.0/20"] + targets = ["notebook-instance"] + rules = [{ protocol = "tcp", ports = [22] }] + } + } +} + +module "cloudnat" { + source = "../../../modules/net-cloudnat" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "${var.prefix}-default" + region = var.region + router_network = module.vpc.0.name +} + +resource "google_project_iam_member" "shared_vpc" { + count = local.use_shared_vpc ? 1 : 0 + project = var.network_config.host_project + role = "roles/compute.networkUser" + member = "serviceAccount:${module.project.service_accounts.robots.notebooks}" +} + + +############################################################################### +# Storage # +############################################################################### + +module "bucket" { + source = "../../../modules/gcs" + project_id = module.project.project_id + prefix = var.prefix + location = var.location + name = "data" + encryption_key = try(local.service_encryption_keys.storage, null) # Example assignment of an encryption key +} + +module "dataset" { + source = "../../../modules/bigquery-dataset" + project_id = module.project.project_id + id = "${replace(var.prefix, "-", "_")}_data" + encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key +} + +############################################################################### +# Vertex AI # +############################################################################### +resource "google_vertex_ai_metadata_store" "store" { + provider = google-beta + project = module.project.project_id + name = "${var.prefix}-metadata-store" + description = "Vertex Ai Metadata Store" + region = var.region + #TODO Check/Implement P4SA logic for IAM role + # encryption_spec { + # kms_key_name = var.service_encryption_keys.ai_metadata_store + # } +} + +module "service-account-notebook" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "notebook-sa" + iam_project_roles = { + (module.project.project_id) = [ + "roles/bigquery.admin", + "roles/bigquery.jobUser", + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/dialogflow.client", + "roles/storage.admin", + ] + } +} + +module "service-account-vertex" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "vertex-sa" + iam_project_roles = { + (module.project.project_id) = [ + "roles/bigquery.admin", + "roles/bigquery.jobUser", + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/dialogflow.client", + "roles/storage.admin", + ] + } +} + +resource "google_notebooks_instance" "playground" { + name = "${var.prefix}-notebook" + location = format("%s-%s", var.region, "b") + machine_type = "e2-medium" + project = module.project.project_id + + container_image { + repository = "gcr.io/deeplearning-platform-release/base-cpu" + tag = "latest" + } + + install_gpu_driver = true + boot_disk_type = "PD_SSD" + boot_disk_size_gb = 110 + disk_encryption = try(local.service_encryption_keys.compute != null, false) ? "CMEK" : null + kms_key = try(local.service_encryption_keys.compute, null) + + no_public_ip = true + no_proxy_access = false + + network = local.vpc + subnet = local.subnet + + service_account = module.service-account-notebook.email + + # Remove once terraform-provider-google/issues/9164 is fixed + lifecycle { + ignore_changes = [disk_encryption, kms_key] + } + + #TODO Uncomment once terraform-provider-google/issues/9273 is fixed + # tags = ["ssh"] + depends_on = [ + google_project_iam_member.shared_vpc, + ] +} diff --git a/blueprints/data-solutions/bq-ml/outputs.tf b/blueprints/data-solutions/bq-ml/outputs.tf new file mode 100644 index 00000000..2b62074b --- /dev/null +++ b/blueprints/data-solutions/bq-ml/outputs.tf @@ -0,0 +1,52 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "bucket" { + description = "GCS Bucket URL." + value = module.bucket.url +} + +output "dataset" { + description = "GCS Bucket URL." + value = module.dataset.id +} + +output "notebook" { + description = "Vertex AI notebook details." + value = { + name = resource.google_notebooks_instance.playground.name + id = resource.google_notebooks_instance.playground.id + } +} + +output "project" { + description = "Project id." + value = module.project.project_id +} + +output "vpc" { + description = "VPC Network." + value = local.vpc +} + +output "service-account-vertex" { + description = "Service account to be used for Vertex AI pipelines" + value = module.service-account-vertex.email +} + +output "vertex-ai-metadata-store" { + description = "" + value = google_vertex_ai_metadata_store.store.id + +} diff --git a/blueprints/data-solutions/bq-ml/variables.tf b/blueprints/data-solutions/bq-ml/variables.tf new file mode 100644 index 00000000..3bd0ca65 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/variables.tf @@ -0,0 +1,69 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "location" { + description = "The location where resources will be deployed." + type = string + default = "EU" +} + +variable "network_config" { + description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." + type = object({ + host_project = string + network_self_link = string + subnet_self_link = string + }) + default = null +} + +variable "prefix" { + description = "Prefix used for resource names." + type = string + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty." + } +} + +variable "project_create" { + description = "Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id." + type = object({ + billing_account_id = string + parent = string + }) + default = null +} + +variable "project_id" { + description = "Project id, references existing project if `project_create` is null." + type = string +} + +variable "region" { + description = "The region where resources will be deployed." + type = string + default = "europe-west1" +} + +variable "service_encryption_keys" { # service encription key + description = "Cloud KMS to use to encrypt different services. Key location should match service region." + type = object({ + bq = string + compute = string + storage = string + }) + default = null +} diff --git a/blueprints/data-solutions/bq-ml/versions.tf b/blueprints/data-solutions/bq-ml/versions.tf new file mode 100644 index 00000000..08492c6f --- /dev/null +++ b/blueprints/data-solutions/bq-ml/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3.1" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.50.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.50.0" # tftest + } + } +} + +