From 4f4a9cd7ac2cee3a1b2e413cd19f6b6b07c35622 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 9 Feb 2022 17:01:25 +0100 Subject: [PATCH] mandatory project creation, refactor --- .../data-platform-foundations/01-landing.tf | 147 ++++++---- .../01-storage-services.tf | 91 ------- .../data-platform-foundations/02-load.tf | 139 ++++++---- .../02-storage-services.tf | 42 --- .../data-platform-foundations/03-composer.tf | 122 +++++---- .../03-orchestration.tf | 206 +++++++------- .../03-storage-services.tf | 27 -- .../04-storage-services.tf | 61 ----- .../04-transformation.tf | 140 +++++++--- .../data-platform-foundations/05-datalake.tf | 251 ++++++++++-------- .../05-storage-services.tf | 95 ------- .../data-platform-foundations/06-common.tf | 43 ++- .../data-platform-foundations/07-exposure.tf | 29 +- .../data-platform-foundations/README.md | 20 +- .../data-platform-foundations/main.tf | 42 +-- .../data-platform-foundations/outputs.tf | 84 +++--- .../data-platform-foundations/variables.tf | 85 ++---- 17 files changed, 684 insertions(+), 940 deletions(-) delete mode 100644 examples/data-solutions/data-platform-foundations/01-storage-services.tf delete mode 100644 examples/data-solutions/data-platform-foundations/02-storage-services.tf delete mode 100644 examples/data-solutions/data-platform-foundations/03-storage-services.tf delete mode 100644 examples/data-solutions/data-platform-foundations/04-storage-services.tf delete mode 100644 examples/data-solutions/data-platform-foundations/05-storage-services.tf diff --git a/examples/data-solutions/data-platform-foundations/01-landing.tf b/examples/data-solutions/data-platform-foundations/01-landing.tf index b67aa648..6a82d4f1 100644 --- a/examples/data-solutions/data-platform-foundations/01-landing.tf +++ b/examples/data-solutions/data-platform-foundations/01-landing.tf @@ -12,67 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -# tfdoc:file:description Landing project. +# tfdoc:file:description land project and resources. locals { - group_iam_lnd = { - "${local.groups.data-engineers}" = [ + land_orch_service_accounts = [ + module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email + ] +} + +module "land-project" { + source = "../../../modules/project" + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "lnd" + group_iam = { + (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", "roles/pubsub.editor", "roles/storage.admin", "roles/storage.objectViewer", ] } - iam_lnd = { - "roles/bigquery.dataEditor" = [ - module.lnd-sa-bq-0.iam_email, - ] - "roles/bigquery.dataViewer" = [ - module.lod-sa-df-0.iam_email, - module.orc-sa-cmp-0.iam_email, - ] - "roles/bigquery.jobUser" = [ - module.orc-sa-cmp-0.iam_email - ] - "roles/bigquery.user" = [ - module.lod-sa-df-0.iam_email - ] - "roles/pubsub.publisher" = [ - module.lnd-sa-ps-0.iam_email - ] - "roles/pubsub.subscriber" = [ - module.lod-sa-df-0.iam_email, - module.orc-sa-cmp-0.iam_email - ] - "roles/storage.objectAdmin" = [ - module.lod-sa-df-0.iam_email, - ] - "roles/storage.objectCreator" = [ - module.lnd-sa-cs-0.iam_email, - ] - "roles/storage.objectViewer" = [ - module.orc-sa-cmp-0.iam_email, - ] - "roles/storage.admin" = [ - module.lod-sa-df-0.iam_email, - ] + iam = { + "roles/bigquery.dataEditor" = [module.land-sa-bq-0.iam_email] + "roles/bigquery.dataViewer" = local.land_orch_service_accounts + "roles/bigquery.jobUser" = [module.orch-sa-cmp-0.iam_email] + "roles/bigquery.user" = [module.load-sa-df-0.iam_email] + "roles/pubsub.publisher" = [module.land-sa-ps-0.iam_email] + "roles/pubsub.subscriber" = local.land_orch_service_accounts + "roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email] + "roles/storage.objectCreator" = [module.land-sa-cs-0.iam_email] + "roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email] + "roles/storage.admin" = [module.load-sa-df-0.iam_email] } - prefix_lnd = "${var.prefix}-lnd" -} - -# Project - -module "lnd-prj" { - source = "../../../modules/project" - name = try(var.project_ids["landing"], "lnd") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["landing"]) - prefix = can(var.project_ids["landing"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_lnd : {} - iam_additive = var.project_create == null ? local.iam_lnd : {} - group_iam = local.group_iam_lnd services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -88,3 +61,73 @@ module "lnd-prj" { storage = [try(local.service_encryption_keys.storage, null)] } } + +# Cloud Storage + +module "land-sa-cs-0" { + source = "../../../modules/iam-service-account" + project_id = module.land-project.project_id + prefix = var.prefix + name = "lnd-cs-0" + iam = { + "roles/iam.serviceAccountTokenCreator" = [ + local.groups_iam.data-engineers + ] + } +} + +module "land-cs-0" { + source = "../../../modules/gcs" + project_id = module.land-project.project_id + prefix = var.prefix + name = "lnd-cs-0" + location = var.region + storage_class = "REGIONAL" + encryption_key = try(local.service_encryption_keys.storage, null) + force_destroy = var.data_force_destroy + # retention_policy = { + # retention_period = 7776000 # 90 * 24 * 60 * 60 + # is_locked = false + # } +} + +# PubSub + +module "land-sa-ps-0" { + source = "../../../modules/iam-service-account" + project_id = module.land-project.project_id + prefix = var.prefix + name = "lnd-ps-0" + iam = { + "roles/iam.serviceAccountTokenCreator" = [ + local.groups_iam.data-engineers + ] + } +} + +module "land-ps-0" { + source = "../../../modules/pubsub" + project_id = module.land-project.project_id + name = "${var.prefix}-lnd-ps-0" + kms_key = try(local.service_encryption_keys.pubsub, null) +} + +# BigQuery + +module "land-sa-bq-0" { + source = "../../../modules/iam-service-account" + project_id = module.land-project.project_id + prefix = var.prefix + name = "lnd-bq-0" + iam = { + "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] + } +} + +module "land-bq-0" { + source = "../../../modules/bigquery-dataset" + project_id = module.land-project.project_id + id = "${replace(var.prefix, "-", "_")}lnd_bq_0" + location = var.region + encryption_key = try(local.service_encryption_keys.bq, null) +} diff --git a/examples/data-solutions/data-platform-foundations/01-storage-services.tf b/examples/data-solutions/data-platform-foundations/01-storage-services.tf deleted file mode 100644 index cf7aba06..00000000 --- a/examples/data-solutions/data-platform-foundations/01-storage-services.tf +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# tfdoc:file:description Landing storage resources (Bigquery, Cloud PubSub, Cloud Storage). - -locals { - lnd_bucket_retention_policy = { - retention_period = 7776000 # 90 * 24 * 60 * 60 - is_locked = false - } -} - -# Cloud Storage resources - -module "lnd-sa-cs-0" { - source = "../../../modules/iam-service-account" - project_id = module.lnd-prj.project_id - name = "cs-0" - prefix = local.prefix_lnd - iam = { - "roles/iam.serviceAccountTokenCreator" = [ - local.groups_iam.data-engineers - ] - } -} - -module "lnd-cs-0" { - source = "../../../modules/gcs" - project_id = module.lnd-prj.project_id - name = "cs-0" - prefix = local.prefix_lnd - location = var.location_config.region - storage_class = "REGIONAL" - # retention_policy = local.lnd_bucket_retention_policy - encryption_key = try(local.service_encryption_keys.storage, null) - force_destroy = var.data_force_destroy -} - -# Cloud PubSub resources - -module "lnd-sa-ps-0" { - source = "../../../modules/iam-service-account" - project_id = module.lnd-prj.project_id - name = "ps-0" - prefix = local.prefix_lnd - iam = { - "roles/iam.serviceAccountTokenCreator" = [ - local.groups_iam.data-engineers - ] - } -} - -module "lnd-ps-0" { - source = "../../../modules/pubsub" - project_id = module.lnd-prj.project_id - name = "${local.prefix_lnd}-ps-0" - kms_key = try(local.service_encryption_keys.pubsub, null) -} - -# Bigquery resources - -module "lnd-sa-bq-0" { - source = "../../../modules/iam-service-account" - project_id = module.lnd-prj.project_id - name = "bq-0" - prefix = local.prefix_lnd - iam = { - "roles/iam.serviceAccountTokenCreator" = [ - local.groups_iam.data-engineers - ] - } -} - -module "lnd-bq-0" { - source = "../../../modules/bigquery-dataset" - project_id = module.lnd-prj.project_id - id = "${replace(local.prefix_lnd, "-", "_")}_bq_0" - location = var.location_config.region - encryption_key = try(local.service_encryption_keys.bq, null) -} diff --git a/examples/data-solutions/data-platform-foundations/02-load.tf b/examples/data-solutions/data-platform-foundations/02-load.tf index 32589cff..4d7169eb 100644 --- a/examples/data-solutions/data-platform-foundations/02-load.tf +++ b/examples/data-solutions/data-platform-foundations/02-load.tf @@ -15,52 +15,53 @@ # tfdoc:file:description Load project and VPC. locals { - group_iam_lod = { - "${local.groups.data-engineers}" = [ + load_service_accounts = [ + "serviceAccount:${module.load-project.service_accounts.robots.dataflow}", + module.load-sa-df-0.iam_email + ] + load_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.load-vpc.0.subnet_self_links)[0] + ) + load_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.load-vpc.0.self_link + ) +} + +# Project + +module "load-project" { + source = "../../../modules/project" + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "lod" + group_iam = { + (local.groups.data-engineers) = [ "roles/compute.viewer", "roles/dataflow.admin", "roles/dataflow.developer", "roles/viewer", ] } - iam_lod = { - "roles/bigquery.jobUser" = [ - module.lod-sa-df-0.iam_email - ] - "roles/compute.serviceAgent" = [ - "serviceAccount:${module.lod-prj.service_accounts.robots.compute}" - ] + iam = { + "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] "roles/dataflow.admin" = [ - module.orc-sa-cmp-0.iam_email, - module.lod-sa-df-0.iam_email - ] - "roles/dataflow.worker" = [ - module.lod-sa-df-0.iam_email - ] - "roles/dataflow.serviceAgent" = [ - "serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}" - ] - "roles/storage.objectAdmin" = [ - "serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}", - module.lod-sa-df-0.iam_email + module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email ] + "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] + "roles/storage.objectAdmin" = local.load_service_accounts + # TODO: these are needed on the shared VPC? + # "roles/compute.serviceAgent" = [ + # "serviceAccount:${module.load-project.service_accounts.robots.compute}" + # ] + # "roles/dataflow.serviceAgent" = [ + # "serviceAccount:${module.load-project.service_accounts.robots.dataflow}" + # ] } - prefix_lod = "${var.prefix}-lod" -} - -# Project - -module "lod-prj" { - source = "../../../modules/project" - name = try(var.project_ids["load"], "lod") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["load"]) - prefix = can(var.project_ids["load"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_lod : {} - iam_additive = var.project_create == null ? local.iam_lod : {} - # group_iam = local.group_iam_lod services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -79,37 +80,67 @@ module "lod-prj" { dataflow = [try(local.service_encryption_keys.dataflow, null)] storage = [try(local.service_encryption_keys.storage, null)] } - shared_vpc_service_config = local._shared_vpc_service_config + shared_vpc_service_config = local.shared_vpc_project == null ? null : { + attach = true + host_project = local.shared_vpc_project + service_identity_iam = { + # TODO: worker service account + "compute.networkUser" = ["dataflow"] + } + } } -module "lod-vpc" { - count = var.network_config.network_self_link != null ? 0 : 1 +module "load-sa-df-0" { + source = "../../../modules/iam-service-account" + project_id = module.load-project.project_id + prefix = var.prefix + name = "load-df-0" + iam = { + "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] + "roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email] + } +} + +module "load-cs-df-0" { + source = "../../../modules/gcs" + project_id = module.load-project.project_id + prefix = var.prefix + name = "load-cs-0" + storage_class = "REGIONAL" + location = var.region + encryption_key = try(local.service_encryption_keys.storage, null) +} + +# internal VPC resources + +module "load-vpc" { source = "../../../modules/net-vpc" - project_id = module.lod-prj.project_id - name = "${local.prefix_lod}-vpc" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.load-project.project_id + name = "{var.prefix}-default" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "${local.prefix_lod}-subnet" - region = var.location_config.region + name = "default" + region = var.region secondary_ip_range = {} } ] } -module "lod-vpc-firewall" { - count = var.network_config.network_self_link != null ? 0 : 1 +module "load-vpc-firewall" { source = "../../../modules/net-vpc-firewall" - project_id = module.lod-prj.project_id - network = local._networks.load.network_name + count = local.use_shared_vpc ? 0 : 1 + project_id = module.load-project.project_id + network = module.load-vpc.0.name admin_ranges = ["10.10.0.0/24"] } -module "lod-nat" { - count = var.network_config.network_self_link != null ? 0 : 1 +module "load-nat" { source = "../../../modules/net-cloudnat" - project_id = module.lod-prj.project_id - region = var.location_config.region - name = "${local.prefix_lod}-default" - router_network = module.lod-vpc[0].name + count = local.use_shared_vpc ? 0 : 1 + project_id = module.load-project.project_id + name = "${var.prefix}-default" + region = var.region + router_network = module.load-vpc.0.name } diff --git a/examples/data-solutions/data-platform-foundations/02-storage-services.tf b/examples/data-solutions/data-platform-foundations/02-storage-services.tf deleted file mode 100644 index d6922c0b..00000000 --- a/examples/data-solutions/data-platform-foundations/02-storage-services.tf +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# tfdoc:file:description Load storage resources (Cloud Storage). - -# Cloud Storage - -module "lod-sa-df-0" { - source = "../../../modules/iam-service-account" - project_id = module.lod-prj.project_id - name = "df-0" - prefix = local.prefix_lod - iam = { - "roles/iam.serviceAccountTokenCreator" = [ - local.groups_iam.data-engineers - ], - "roles/iam.serviceAccountUser" = [ - module.orc-sa-cmp-0.iam_email, - ] - } -} - -module "lod-cs-df-0" { - source = "../../../modules/gcs" - project_id = module.lod-prj.project_id - name = "cs-0" - prefix = local.prefix_lod - storage_class = "REGIONAL" - location = var.location_config.region - encryption_key = try(local.service_encryption_keys.storage, null) -} diff --git a/examples/data-solutions/data-platform-foundations/03-composer.tf b/examples/data-solutions/data-platform-foundations/03-composer.tf index e0ec0a69..ee79c137 100644 --- a/examples/data-solutions/data-platform-foundations/03-composer.tf +++ b/examples/data-solutions/data-platform-foundations/03-composer.tf @@ -14,86 +14,96 @@ # tfdoc:file:description Orchestration Cloud Composer definition. -module "orc-sa-cmp-0" { +module "orch-sa-cmp-0" { source = "../../../modules/iam-service-account" - project_id = module.orc-prj.project_id - name = "cmp-0" - prefix = local.prefix_orc + project_id = module.orch-project.project_id + prefix = var.prefix + name = "orc-cmp-0" iam = { - "roles/iam.serviceAccountTokenCreator" = [ - local.groups_iam.data-engineers - ], - "roles/iam.serviceAccountUser" = [ - module.orc-sa-cmp-0.iam_email, - ] + "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] + "roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email] } } -resource "google_composer_environment" "orc-cmp-0" { - name = "${local.prefix_orc}-cmp-0" - region = var.location_config.region +resource "google_composer_environment" "orch-cmp-0" { provider = google-beta - project = module.orc-prj.project_id + project = module.orch-project.project_id + name = "${var.prefix}-orc-cmp-0" + region = var.region config { node_count = var.composer_config.node_count node_config { - zone = "${var.location_config.region}-b" - service_account = module.orc-sa-cmp-0.email - network = local._networks.orchestration.network - subnetwork = local._networks.orchestration.subnet + zone = "${var.region}-b" + service_account = module.orch-sa-cmp-0.email + network = local.orch_vpc + subnetwork = local.orch_subnet tags = ["composer-worker", "http-server", "https-server"] ip_allocation_policy { - use_ip_aliases = "true" - cluster_secondary_range_name = try(var.network_config.composer_secondary_ranges.pods, "pods") - services_secondary_range_name = try(var.network_config.composer_secondary_ranges.services, "services") + use_ip_aliases = "true" + cluster_secondary_range_name = try( + var.network_config.composer_secondary_ranges.pods, "pods" + ) + services_secondary_range_name = try( + var.network_config.composer_secondary_ranges.services, "services" + ) } } software_config { image_version = var.composer_config.airflow_version env_variables = merge( var.composer_config.env_variables, { - DTL_L0_PRJ = module.dtl-0-prj.project_id - DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id - DTL_L0_GCS = module.dtl-0-cs-0.url - DTL_L1_PRJ = module.dtl-1-prj.project_id - DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id - DTL_L1_GCS = module.dtl-1-cs-0.url - DTL_L2_PRJ = module.dtl-2-prj.project_id - DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id - DTL_L2_GCS = module.dtl-2-cs-0.url - DTL_PLG_PRJ = module.dtl-plg-prj.project_id - DTL_PLG_BQ_DATASET = module.dtl-plg-bq-0.dataset_id - DTL_PLG_GCS = module.dtl-plg-cs-0.url - GCP_REGION = var.location_config.region - LND_PRJ = module.lnd-prj.project_id - LND_BQ = module.lnd-bq-0.dataset_id - LND_GCS = module.lnd-cs-0.url - LND_PS = module.lnd-ps-0.id - LOD_PRJ = module.lod-prj.project_id - LOD_GCS_STAGING = module.lod-cs-df-0.url - LOD_NET_VPC = local._networks.load.network - LOD_NET_SUBNET = local._networks.load.subnet - LOD_SA_DF = module.lod-sa-df-0.email - ORC_PRJ = module.orc-prj.project_id - ORC_GCS = module.orc-cs-0.url - TRF_PRJ = module.trf-prj.project_id - TRF_GCS_STAGING = module.trf-cs-df-0.url - TRF_NET_VPC = local._networks.transformation.network - TRF_NET_SUBNET = local._networks.transformation.subnet - TRF_SA_DF = module.trf-sa-df-0.email - TRF_SA_BQ = module.trf-sa-bq-0.email + DTL_L0_PRJ = module.lake-0-project.project_id + DTL_L0_BQ_DATASET = module.lake-0-bq-0.dataset_id + DTL_L0_GCS = module.lake-0-cs-0.url + DTL_L1_PRJ = module.lake-1-project.project_id + DTL_L1_BQ_DATASET = module.lake-1-bq-0.dataset_id + DTL_L1_GCS = module.lake-1-cs-0.url + DTL_L2_PRJ = module.lake-2-project.project_id + DTL_L2_BQ_DATASET = module.lake-2-bq-0.dataset_id + DTL_L2_GCS = module.lake-2-cs-0.url + DTL_PLG_PRJ = module.lake-plg-project.project_id + DTL_PLG_BQ_DATASET = module.lake-plg-bq-0.dataset_id + DTL_PLG_GCS = module.lake-plg-cs-0.url + GCP_REGION = var.region + LND_PRJ = module.land-project.project_id + LND_BQ = module.land-bq-0.dataset_id + LND_GCS = module.land-cs-0.url + LND_PS = module.land-ps-0.id + LOD_PRJ = module.load-project.project_id + LOD_GCS_STAGING = module.load-cs-df-0.url + LOD_NET_VPC = local.load_vpc + LOD_NET_SUBNET = local.load_subnet + LOD_SA_DF = module.load-sa-df-0.email + ORC_PRJ = module.orch-project.project_id + ORC_GCS = module.orch-cs-0.url + TRF_PRJ = module.transf-project.project_id + TRF_GCS_STAGING = module.transf-cs-df-0.url + TRF_NET_VPC = local.transf_vpc + TRF_NET_SUBNET = local.transf_subnet + TRF_SA_DF = module.transf-sa-df-0.email + TRF_SA_BQ = module.transf-sa-bq-0.email } ) } private_environment_config { - enable_private_endpoint = "true" - cloud_sql_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.cloudsql, "10.20.10.0/24") - master_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28") - web_server_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28") + enable_private_endpoint = "true" + cloud_sql_ipv4_cidr_block = try( + var.network_config.composer_ip_ranges.cloudsql, "10.20.10.0/24" + ) + master_ipv4_cidr_block = try( + var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28" + ) + web_server_ipv4_cidr_block = try( + var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28" + ) } dynamic "encryption_config" { - for_each = try(local.service_encryption_keys.composer != null, false) ? { 1 = 1 } : {} + for_each = ( + try(local.service_encryption_keys.composer != null, false) + ? { 1 = 1 } + : {} + ) content { kms_key_name = try(local.service_encryption_keys.composer, null) } diff --git a/examples/data-solutions/data-platform-foundations/03-orchestration.tf b/examples/data-solutions/data-platform-foundations/03-orchestration.tf index 3f38eeb8..0863256f 100644 --- a/examples/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/examples/data-solutions/data-platform-foundations/03-orchestration.tf @@ -15,8 +15,26 @@ # tfdoc:file:description Orchestration project and VPC. locals { - group_iam_orc = { - "${local.groups.data-engineers}" = [ + orch_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.orch-vpc.0.subnet_self_links)[0] + ) + orch_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.orch-vpc.0.self_link + ) +} + +module "orch-project" { + source = "../../../modules/project" + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "orc" + group_iam = { + (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", "roles/bigquery.jobUser", "roles/cloudbuild.builds.editor", @@ -30,98 +48,99 @@ locals { "roles/compute.networkUser" ] } - - iam_orc = { + iam = { "roles/bigquery.dataEditor" = [ - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email, - module.orc-sa-cmp-0.iam_email, + module.load-sa-df-0.iam_email, + module.transf-sa-df-0.iam_email, + module.orch-sa-cmp-0.iam_email, ] "roles/bigquery.jobUser" = [ - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email, - module.orc-sa-cmp-0.iam_email, + module.load-sa-df-0.iam_email, + module.transf-sa-df-0.iam_email, + module.orch-sa-cmp-0.iam_email, ] "roles/composer.worker" = [ - module.orc-sa-cmp-0.iam_email - ] - "roles/compute.networkUser" = [ - module.orc-sa-cmp-0.iam_email, - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email, - "serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}", - "serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}", - "serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}", - "serviceAccount:${module.orc-prj.service_accounts.cloud_services}" + module.orch-sa-cmp-0.iam_email ] "roles/iam.serviceAccountUser" = [ - module.orc-sa-cmp-0.iam_email, + module.orch-sa-cmp-0.iam_email ] "roles/storage.objectAdmin" = [ - module.lod-sa-df-0.iam_email, - module.orc-sa-cmp-0.iam_email, - "serviceAccount:${module.orc-prj.service_accounts.robots.composer}", + module.load-sa-df-0.iam_email, + module.orch-sa-cmp-0.iam_email, + "serviceAccount:${module.orch-project.service_accounts.robots.composer}", ] "roles/storage.admin" = [ - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email + module.load-sa-df-0.iam_email, + module.transf-sa-df-0.iam_email ] } - - prefix_orc = "${var.prefix}-orc" -} - -module "orc-prj" { - source = "../../../modules/project" - name = try(var.project_ids["orchestration"], "orc") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["orchestration"]) - prefix = can(var.project_ids["orchestration"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_orc : {} - iam_additive = var.project_create == null ? local.iam_orc : {} - group_iam = local.group_iam_orc - oslogin = false + oslogin = false policy_boolean = { "constraints/compute.requireOsLogin" = false } - services = concat( - var.project_services, - [ - "artifactregistry.googleapis.com", - "bigquery.googleapis.com", - "bigqueryreservation.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudbuild.googleapis.com", - "cloudkms.googleapis.com", - "composer.googleapis.com", - "compute.googleapis.com", - "container.googleapis.com", - "containerregistry.googleapis.com", - "dataflow.googleapis.com", - "pubsub.googleapis.com", - "servicenetworking.googleapis.com", - "storage.googleapis.com", - "storage-component.googleapis.com" + services = concat(var.project_services, [ + "artifactregistry.googleapis.com", + "bigquery.googleapis.com", + "bigqueryreservation.googleapis.com", + "bigquerystorage.googleapis.com", + "cloudbuild.googleapis.com", + "cloudkms.googleapis.com", + "composer.googleapis.com", + "compute.googleapis.com", + "container.googleapis.com", + "containerregistry.googleapis.com", + "dataflow.googleapis.com", + "pubsub.googleapis.com", + "servicenetworking.googleapis.com", + "storage.googleapis.com", + "storage-component.googleapis.com" ]) service_encryption_key_ids = { composer = [try(local.service_encryption_keys.composer, null)] storage = [try(local.service_encryption_keys.storage, null)] } - shared_vpc_service_config = local._shared_vpc_service_config + shared_vpc_service_config = local.shared_vpc_project == null ? null : { + attach = true + host_project = local.shared_vpc_project + service_identity_iam = { + "roles/composer.sharedVpcAgent" = [ + "composer" + ] + "roles/compute.networkUser" = [ + "cloudservices", "container-engine", "dataflow" + ] + "roles/container.hostServiceAgentUser" = [ + "container-engine" + ] + } + } } -module "orc-vpc" { - count = var.network_config.network_self_link != null ? 0 : 1 +# Cloud Storage + +module "orch-cs-0" { + source = "../../../modules/gcs" + project_id = module.orch-project.project_id + prefix = var.prefix + name = "orc-cs-0" + location = var.region + storage_class = "REGIONAL" + encryption_key = try(local.service_encryption_keys.storage, null) +} + +# internal VPC resources + +module "orch-vpc" { source = "../../../modules/net-vpc" - project_id = module.orc-prj.project_id - name = "${local.prefix_orc}-vpc" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.orch-project.project_id + name = "${var.prefix}-default" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "${local.prefix_orc}-subnet" - region = var.location_config.region + name = "default" + region = var.region secondary_ip_range = { pods = "10.10.8.0/22" services = "10.10.12.0/24" @@ -130,52 +149,19 @@ module "orc-vpc" { ] } -resource "google_project_iam_binding" "composer_shared_vpc_agent" { - count = var.network_config.network_self_link != null ? 1 : 0 - project = local._shared_vpc_project - role = "roles/composer.sharedVpcAgent" - members = [ - "serviceAccount:${module.orc-prj.service_accounts.robots.composer}" - ] -} - -resource "google_project_iam_binding" "gke_host_service_agent_user" { - count = var.network_config.network_self_link != null ? 1 : 0 - project = local._shared_vpc_project - role = "roles/container.hostServiceAgentUser" - members = [ - "serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}" - ] -} - -resource "google_project_iam_binding" "composer_network_user_agent" { - count = var.network_config.network_self_link != null ? 1 : 0 - project = local._shared_vpc_project - role = "roles/compute.networkUser" - members = [ - module.orc-sa-cmp-0.iam_email, - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email, - "serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}", - "serviceAccount:${module.orc-prj.service_accounts.cloud_services}", - "serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}", - "serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}", - ] -} - -module "orc-vpc-firewall" { - count = var.network_config.network_self_link != null ? 0 : 1 +module "orch-vpc-firewall" { source = "../../../modules/net-vpc-firewall" - project_id = module.orc-prj.project_id - network = local._networks.orchestration.network_name + count = local.use_shared_vpc ? 0 : 1 + project_id = module.orch-project.project_id + network = module.orch-vpc.0.name admin_ranges = ["10.10.0.0/24"] } -module "orc-nat" { - count = var.network_config.network_self_link != null ? 0 : 1 +module "orch-nat" { + count = local.use_shared_vpc ? 0 : 1 source = "../../../modules/net-cloudnat" - project_id = module.orc-prj.project_id - region = var.location_config.region - name = "${local.prefix_orc}-default" - router_network = local._networks.orchestration.network_name + project_id = module.orch-project.project_id + name = "${var.prefix}-default" + region = var.region + router_network = module.orch-vpc.0.name } diff --git a/examples/data-solutions/data-platform-foundations/03-storage-services.tf b/examples/data-solutions/data-platform-foundations/03-storage-services.tf deleted file mode 100644 index efc674e4..00000000 --- a/examples/data-solutions/data-platform-foundations/03-storage-services.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# tfdoc:file:description Orchestration storage resources (Cloud Storage). - -# Cloud Storage - -module "orc-cs-0" { - source = "../../../modules/gcs" - project_id = module.orc-prj.project_id - name = "cs-0" - prefix = local.prefix_orc - location = var.location_config.region - storage_class = "REGIONAL" - encryption_key = try(local.service_encryption_keys.storage, null) -} diff --git a/examples/data-solutions/data-platform-foundations/04-storage-services.tf b/examples/data-solutions/data-platform-foundations/04-storage-services.tf deleted file mode 100644 index ea451cbc..00000000 --- a/examples/data-solutions/data-platform-foundations/04-storage-services.tf +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# tfdoc:file:description Landing storage resources (Bigquery, Cloud Storage). - -# Cloud Storage - -module "trf-sa-df-0" { - source = "../../../modules/iam-service-account" - project_id = module.trf-prj.project_id - name = "df-0" - prefix = local.prefix_trf - iam = { - "roles/iam.serviceAccountTokenCreator" = [ - local.groups_iam.data-engineers, - module.orc-sa-cmp-0.iam_email, - ], - "roles/iam.serviceAccountUser" = [ - module.orc-sa-cmp-0.iam_email, - ] - } -} - -module "trf-cs-df-0" { - source = "../../../modules/gcs" - project_id = module.trf-prj.project_id - name = "cs-0" - prefix = local.prefix_trf - location = var.location_config.region - storage_class = "REGIONAL" - encryption_key = try(local.service_encryption_keys.storage, null) -} - -# Bigquery - -module "trf-sa-bq-0" { - source = "../../../modules/iam-service-account" - project_id = module.trf-prj.project_id - name = "bq-0" - prefix = local.prefix_trf - iam = { - "roles/iam.serviceAccountTokenCreator" = [ - local.groups_iam.data-engineers, - module.orc-sa-cmp-0.iam_email, - ], - "roles/iam.serviceAccountUser" = [ - module.orc-sa-cmp-0.iam_email, - ] - } -} diff --git a/examples/data-solutions/data-platform-foundations/04-transformation.tf b/examples/data-solutions/data-platform-foundations/04-transformation.tf index e048171a..ed254ee4 100644 --- a/examples/data-solutions/data-platform-foundations/04-transformation.tf +++ b/examples/data-solutions/data-platform-foundations/04-transformation.tf @@ -15,47 +15,49 @@ # tfdoc:file:description Trasformation project and VPC. locals { - group_iam_trf = { - "${local.groups.data-engineers}" = [ + transf_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.transf-vpc.0.subnet_self_links)[0] + ) + transf_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.transf-vpc.0.self_link + ) +} + +module "transf-project" { + source = "../../../modules/project" + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "trf" + group_iam = { + (local.groups.data-engineers) = [ "roles/bigquery.jobUser", "roles/dataflow.admin", ] } - iam_trf = { + iam = { "roles/bigquery.dataViewer" = [ - module.orc-sa-cmp-0.iam_email + module.orch-sa-cmp-0.iam_email ] "roles/bigquery.jobUser" = [ - module.trf-sa-bq-0.iam_email, + module.transf-sa-bq-0.iam_email, ] "roles/dataflow.admin" = [ - module.orc-sa-cmp-0.iam_email, + module.orch-sa-cmp-0.iam_email, ] "roles/dataflow.worker" = [ - module.trf-sa-df-0.iam_email + module.transf-sa-df-0.iam_email ] "roles/storage.objectAdmin" = [ - module.trf-sa-df-0.iam_email, - module.orc-sa-cmp-0.iam_email, - "serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}" + module.transf-sa-df-0.iam_email, + module.orch-sa-cmp-0.iam_email, + "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" ] } - prefix_trf = "${var.prefix}-trf" -} - -# Project - -module "trf-prj" { - source = "../../../modules/project" - name = try(var.project_ids["trasformation"], "trf") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["trasformation"]) - prefix = can(var.project_ids["trasformation"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_trf : {} - iam_additive = var.project_create == null ? local.iam_trf : {} - group_iam = local.group_iam_trf services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -73,37 +75,89 @@ module "trf-prj" { dataflow = [try(local.service_encryption_keys.dataflow, null)] storage = [try(local.service_encryption_keys.storage, null)] } - shared_vpc_service_config = local._shared_vpc_service_config + shared_vpc_service_config = local.shared_vpc_project == null ? null : { + attach = true + host_project = local.shared_vpc_project + service_identity_iam = {} + } } -module "trf-vpc" { - count = var.network_config.network_self_link != null ? 0 : 1 +# Cloud Storage + +module "transf-sa-df-0" { + source = "../../../modules/iam-service-account" + project_id = module.transf-project.project_id + prefix = var.prefix + name = "trf-df-0" + iam = { + "roles/iam.serviceAccountTokenCreator" = [ + local.groups_iam.data-engineers, + module.orch-sa-cmp-0.iam_email + ], + "roles/iam.serviceAccountUser" = [ + module.orch-sa-cmp-0.iam_email + ] + } +} + +module "transf-cs-df-0" { + source = "../../../modules/gcs" + project_id = module.transf-project.project_id + prefix = var.prefix + name = "trf-cs-0" + location = var.region + storage_class = "REGIONAL" + encryption_key = try(local.service_encryption_keys.storage, null) +} + +# BigQuery + +module "transf-sa-bq-0" { + source = "../../../modules/iam-service-account" + project_id = module.transf-project.project_id + prefix = var.prefix + name = "trf-bq-0" + iam = { + "roles/iam.serviceAccountTokenCreator" = [ + local.groups_iam.data-engineers, + module.orch-sa-cmp-0.iam_email + ], + "roles/iam.serviceAccountUser" = [ + module.orch-sa-cmp-0.iam_email + ] + } +} + +# internal VPC resources + +module "transf-vpc" { source = "../../../modules/net-vpc" - project_id = module.trf-prj.project_id - name = "${local.prefix_trf}-vpc" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.transf-project.project_id + name = "{var.prefix}-default" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "${local.prefix_trf}-subnet" - region = var.location_config.region + name = "default" + region = var.region secondary_ip_range = {} } ] } -module "trf-vpc-firewall" { - count = var.network_config.network_self_link != null ? 0 : 1 +module "transf-vpc-firewall" { source = "../../../modules/net-vpc-firewall" - project_id = module.trf-prj.project_id - network = local._networks.transformation.network_name + count = local.use_shared_vpc ? 0 : 1 + project_id = module.transf-project.project_id + network = module.transf-vpc.0.name admin_ranges = ["10.10.0.0/24"] } -module "trf-nat" { - count = var.network_config.network_self_link != null ? 0 : 1 +module "transf-nat" { source = "../../../modules/net-cloudnat" - project_id = module.trf-prj.project_id - region = var.location_config.region - name = "${local.prefix_trf}-default" - router_network = module.trf-vpc[0].name + count = local.use_shared_vpc ? 0 : 1 + project_id = module.transf-project.project_id + name = "${var.prefix}-default" + region = var.region + router_network = module.transf-vpc.0.name } diff --git a/examples/data-solutions/data-platform-foundations/05-datalake.tf b/examples/data-solutions/data-platform-foundations/05-datalake.tf index b2b9bddc..71ad0447 100644 --- a/examples/data-solutions/data-platform-foundations/05-datalake.tf +++ b/examples/data-solutions/data-platform-foundations/05-datalake.tf @@ -15,12 +15,12 @@ # tfdoc:file:description Datalake projects. locals { - group_iam_dtl = { - "${local.groups.data-engineers}" = [ + lake_group_iam = { + (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", "roles/storage.admin", ], - "${local.groups.data-analysts}" = [ + (local.groups.data-analysts) = [ "roles/bigquery.dataViewer", "roles/bigquery.jobUser", "roles/bigquery.user", @@ -29,150 +29,185 @@ locals { "roles/storage.objectViewer", ] } - iam_dtl = { + lake_iam = { "roles/bigquery.dataEditor" = [ - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email, - module.trf-sa-bq-0.iam_email, - module.orc-sa-cmp-0.iam_email, + module.load-sa-df-0.iam_email, + module.transf-sa-df-0.iam_email, + module.transf-sa-bq-0.iam_email, + module.orch-sa-cmp-0.iam_email, ] "roles/bigquery.jobUser" = [ - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email, + module.load-sa-df-0.iam_email, + module.transf-sa-df-0.iam_email, ] "roles/storage.admin" = [ - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email, + module.load-sa-df-0.iam_email, + module.transf-sa-df-0.iam_email, ] "roles/storage.objectCreator" = [ - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email, - module.trf-sa-bq-0.iam_email, - module.orc-sa-cmp-0.iam_email, + module.load-sa-df-0.iam_email, + module.transf-sa-df-0.iam_email, + module.transf-sa-bq-0.iam_email, + module.orch-sa-cmp-0.iam_email, ] "roles/storage.objectViewer" = [ - module.trf-sa-df-0.iam_email, - module.trf-sa-bq-0.iam_email, - module.orc-sa-cmp-0.iam_email, + module.transf-sa-df-0.iam_email, + module.transf-sa-bq-0.iam_email, + module.orch-sa-cmp-0.iam_email, ] } - prefix_dtl = "${var.prefix}-dtl" + lake_services = concat(var.project_services, [ + "bigquery.googleapis.com", + "bigqueryreservation.googleapis.com", + "bigquerystorage.googleapis.com", + "cloudkms.googleapis.com", + "compute.googleapis.com", + "dataflow.googleapis.com", + "pubsub.googleapis.com", + "servicenetworking.googleapis.com", + "storage.googleapis.com", + "storage-component.googleapis.com" + ]) } # Project -module "dtl-0-prj" { +module "lake-0-project" { source = "../../../modules/project" - name = try(var.project_ids["datalake-l0"], "dtl-0") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["datalake-l0"]) - prefix = can(var.project_ids["datalake-l0"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_dtl : {} - iam_additive = var.project_create == null ? local.iam_dtl : {} - group_iam = local.group_iam_dtl - services = concat(var.project_services, [ - "bigquery.googleapis.com", - "bigqueryreservation.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudkms.googleapis.com", - "compute.googleapis.com", - "dataflow.googleapis.com", - "pubsub.googleapis.com", - "servicenetworking.googleapis.com", - "storage.googleapis.com", - "storage-component.googleapis.com" - ]) + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "dtl-0" + group_iam = local.lake_group_iam + iam = local.lake_iam + services = local.lake_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] } } -module "dtl-1-prj" { +module "lake-1-project" { source = "../../../modules/project" - name = try(var.project_ids["datalake-l1"], "dtl-1") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["datalake-l1"]) - prefix = can(var.project_ids["datalake-l1"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_dtl : {} - iam_additive = var.project_create == null ? local.iam_dtl : {} - group_iam = local.group_iam_dtl - services = concat(var.project_services, [ - "bigquery.googleapis.com", - "bigqueryreservation.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudkms.googleapis.com", - "compute.googleapis.com", - "dataflow.googleapis.com", - "pubsub.googleapis.com", - "servicenetworking.googleapis.com", - "storage.googleapis.com", - "storage-component.googleapis.com" - ]) + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "dtl-1" + group_iam = local.lake_group_iam + iam = local.lake_iam + services = local.lake_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] } } -module "dtl-2-prj" { +module "lake-2-project" { source = "../../../modules/project" - name = try(var.project_ids["datalake-l2"], "dtl-2") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["datalake-l2"]) - prefix = can(var.project_ids["datalake-l2"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_dtl : {} - iam_additive = var.project_create == null ? local.iam_dtl : {} - group_iam = local.group_iam_dtl - services = concat(var.project_services, [ - "bigquery.googleapis.com", - "bigqueryreservation.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudkms.googleapis.com", - "compute.googleapis.com", - "dataflow.googleapis.com", - "pubsub.googleapis.com", - "servicenetworking.googleapis.com", - "storage.googleapis.com", - "storage-component.googleapis.com" - ]) + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "dtl-2" + group_iam = local.lake_group_iam + iam = local.lake_iam + services = local.lake_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] } } -module "dtl-plg-prj" { +module "lake-plg-project" { source = "../../../modules/project" - name = try(var.project_ids["datalake-playground"], "dtl-plg") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["datalake-playground"]) - prefix = can(var.project_ids["datalake-playground"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_dtl : {} - iam_additive = var.project_create == null ? local.iam_dtl : {} - group_iam = local.group_iam_dtl - services = concat(var.project_services, [ - "bigquery.googleapis.com", - "bigqueryreservation.googleapis.com", - "bigquerystorage.googleapis.com", - "cloudkms.googleapis.com", - "compute.googleapis.com", - "dataflow.googleapis.com", - "pubsub.googleapis.com", - "servicenetworking.googleapis.com", - "storage.googleapis.com", - "storage-component.googleapis.com" - ]) + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "dtl-plg" + group_iam = local.lake_group_iam + iam = local.lake_iam + services = local.lake_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] } } + +# Bigquery + +module "lake-0-bq-0" { + source = "../../../modules/bigquery-dataset" + project_id = module.lake-0-project.project_id + id = "${replace(var.prefix, "-", "_")}_dtl_0_bq_0" + location = var.region + encryption_key = try(local.service_encryption_keys.bq, null) +} + +module "lake-1-bq-0" { + source = "../../../modules/bigquery-dataset" + project_id = module.lake-1-project.project_id + id = "${replace(var.prefix, "-", "_")}_dtl_1_bq_0" + location = var.region + encryption_key = try(local.service_encryption_keys.bq, null) +} + +module "lake-2-bq-0" { + source = "../../../modules/bigquery-dataset" + project_id = module.lake-2-project.project_id + id = "${replace(var.prefix, "-", "_")}_dtl_2_bq_0" + location = var.region + encryption_key = try(local.service_encryption_keys.bq, null) +} + +module "lake-plg-bq-0" { + source = "../../../modules/bigquery-dataset" + project_id = module.lake-plg-project.project_id + id = "${replace(var.prefix, "-", "_")}_dtl_plg_bq_0" + location = var.region + encryption_key = try(local.service_encryption_keys.bq, null) +} + +# Cloud storage + +module "lake-0-cs-0" { + source = "../../../modules/gcs" + project_id = module.lake-0-project.project_id + prefix = var.prefix + name = "dtl-0-cs-0" + location = var.region + storage_class = "REGIONAL" + encryption_key = try(local.service_encryption_keys.storage, null) + force_destroy = var.data_force_destroy +} + +module "lake-1-cs-0" { + source = "../../../modules/gcs" + project_id = module.lake-1-project.project_id + prefix = var.prefix + name = "dtl-1-cs-0" + location = var.region + storage_class = "REGIONAL" + encryption_key = try(local.service_encryption_keys.storage, null) + force_destroy = var.data_force_destroy +} + +module "lake-2-cs-0" { + source = "../../../modules/gcs" + project_id = module.lake-2-project.project_id + prefix = var.prefix + name = "dtl-2-cs-0" + location = var.region + storage_class = "REGIONAL" + encryption_key = try(local.service_encryption_keys.storage, null) + force_destroy = var.data_force_destroy +} + +module "lake-plg-cs-0" { + source = "../../../modules/gcs" + project_id = module.lake-plg-project.project_id + prefix = var.prefix + name = "dtl-plg-cs-0" + location = var.region + storage_class = "REGIONAL" + encryption_key = try(local.service_encryption_keys.storage, null) + force_destroy = var.data_force_destroy +} diff --git a/examples/data-solutions/data-platform-foundations/05-storage-services.tf b/examples/data-solutions/data-platform-foundations/05-storage-services.tf deleted file mode 100644 index 1a3e879e..00000000 --- a/examples/data-solutions/data-platform-foundations/05-storage-services.tf +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# tfdoc:file:description Datalake storage resources (Bigquery, Cloud Storage). - -# Bigquery - -module "dtl-0-bq-0" { - source = "../../../modules/bigquery-dataset" - project_id = module.dtl-0-prj.project_id - id = "${replace(local.prefix_dtl, "-", "_")}_0_bq_0" - location = var.location_config.region - encryption_key = try(local.service_encryption_keys.bq, null) -} - -module "dtl-1-bq-0" { - source = "../../../modules/bigquery-dataset" - project_id = module.dtl-1-prj.project_id - id = "${replace(local.prefix_dtl, "-", "_")}_1_bq_0" - location = var.location_config.region - encryption_key = try(local.service_encryption_keys.bq, null) -} - -module "dtl-2-bq-0" { - source = "../../../modules/bigquery-dataset" - project_id = module.dtl-2-prj.project_id - id = "${replace(local.prefix_dtl, "-", "_")}_2_bq_0" - location = var.location_config.region - encryption_key = try(local.service_encryption_keys.bq, null) -} - -module "dtl-plg-bq-0" { - source = "../../../modules/bigquery-dataset" - project_id = module.dtl-plg-prj.project_id - id = "${replace(local.prefix_dtl, "-", "_")}_plg_bq_0" - location = var.location_config.region - encryption_key = try(local.service_encryption_keys.bq, null) -} - -# Cloud storage - -module "dtl-0-cs-0" { - source = "../../../modules/gcs" - project_id = module.dtl-0-prj.project_id - name = "0-cs-0" - prefix = local.prefix_dtl - location = var.location_config.region - storage_class = "REGIONAL" - encryption_key = try(local.service_encryption_keys.storage, null) - force_destroy = var.data_force_destroy -} - -module "dtl-1-cs-0" { - source = "../../../modules/gcs" - project_id = module.dtl-1-prj.project_id - name = "1-cs-0" - prefix = local.prefix_dtl - location = var.location_config.region - storage_class = "REGIONAL" - encryption_key = try(local.service_encryption_keys.storage, null) - force_destroy = var.data_force_destroy -} - -module "dtl-2-cs-0" { - source = "../../../modules/gcs" - project_id = module.dtl-2-prj.project_id - name = "2-cs-0" - prefix = local.prefix_dtl - location = var.location_config.region - storage_class = "REGIONAL" - encryption_key = try(local.service_encryption_keys.storage, null) - force_destroy = var.data_force_destroy -} - -module "dtl-plg-cs-0" { - source = "../../../modules/gcs" - project_id = module.dtl-plg-prj.project_id - name = "plg-cs-0" - prefix = local.prefix_dtl - location = var.location_config.region - storage_class = "REGIONAL" - encryption_key = try(local.service_encryption_keys.storage, null) - force_destroy = var.data_force_destroy -} diff --git a/examples/data-solutions/data-platform-foundations/06-common.tf b/examples/data-solutions/data-platform-foundations/06-common.tf index 68957df6..126f947e 100644 --- a/examples/data-solutions/data-platform-foundations/06-common.tf +++ b/examples/data-solutions/data-platform-foundations/06-common.tf @@ -14,39 +14,28 @@ # tfdoc:file:description common project. -locals { - group_iam_cmn = { - "${local.groups.data-engineers}" = [ +module "common-project" { + source = "../../../modules/project" + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "cmn" + group_iam = { + (local.groups.data-engineers) = [ "roles/dlp.reader", "roles/dlp.user", "roles/dlp.estimatesAdmin", - ], - "${local.groups.data-security}" = [ + ] + (local.groups.data-security) = [ "roles/dlp.admin", - ], - } - iam_cmn = { - "roles/dlp.user" = [ - module.lod-sa-df-0.iam_email, - module.trf-sa-df-0.iam_email ] } - prefix_cmn = "${var.prefix}-cmn" -} - -# Project - -module "cmn-prj" { - source = "../../../modules/project" - name = try(var.project_ids["common"], "cmn") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["common"]) - prefix = can(var.project_ids["common"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_cmn : {} - iam_additive = var.project_create == null ? local.iam_cmn : {} - group_iam = local.group_iam_cmn + iam = { + "roles/dlp.user" = [ + module.load-sa-df-0.iam_email, + module.transf-sa-df-0.iam_email + ] + } services = concat(var.project_services, [ "datacatalog.googleapis.com", "dlp.googleapis.com", diff --git a/examples/data-solutions/data-platform-foundations/07-exposure.tf b/examples/data-solutions/data-platform-foundations/07-exposure.tf index aea382d1..8fa819f6 100644 --- a/examples/data-solutions/data-platform-foundations/07-exposure.tf +++ b/examples/data-solutions/data-platform-foundations/07-exposure.tf @@ -14,29 +14,10 @@ # tfdoc:file:description common project. -locals { - group_iam_exp = { - #TODO add group => role mapping to asign on exposure project - } - iam_exp = { - #TODO add role => service account mapping to assign roles on exposure project - } - prefix_exp = "${var.prefix}-exp" -} - -# Project - -module "exp-prj" { +module "exp-project" { source = "../../../modules/project" - name = try(var.project_ids["exposure"], "exp") - parent = try(var.project_create.parent, null) - billing_account = try(var.project_create.billing_account_id, null) - project_create = can(var.project_ids["exposure"]) - prefix = can(var.project_ids["exposure"]) ? var.prefix : null - # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_exp : {} - iam_additive = var.project_create == null ? local.iam_exp : {} - group_iam = local.group_iam_exp - services = concat(var.project_services, [ - ]) + parent = var.folder_id + billing_account = var.billing_account_id + prefix = var.prefix + name = "exp" } diff --git a/examples/data-solutions/data-platform-foundations/README.md b/examples/data-solutions/data-platform-foundations/README.md index 9c6c722c..2ac06df6 100644 --- a/examples/data-solutions/data-platform-foundations/README.md +++ b/examples/data-solutions/data-platform-foundations/README.md @@ -241,16 +241,16 @@ Description of commands: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [network_config](variables.tf#L47) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…} | ✓ | | -| [organization](variables.tf#L75) | Organization details. | object({…}) | ✓ | | -| [prefix](variables.tf#L82) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | -| [composer_config](variables.tf#L18) | | object({…}) | | {…} | -| [data_force_destroy](variables.tf#L31) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [groups](variables.tf#L37) | Groups. | map(string) | | {…} | -| [location_config](variables.tf#L135) | Locations where resources will be deployed. Map to configure region and multiregion specs. | object({…}) | | {…} | -| [project_create](variables.tf#L87) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [project_ids](variables.tf#L96) | Project id, references existing project if `project_create` is null. | object({…}) | | {…} | -| [project_services](variables.tf#L124) | List of core services enabled on all projects. | list(string) | | […] | +| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | +| [folder_id](variables.tf#L41) | Folder to be used for the networking resources in folders/nnnn format. | string | ✓ | | +| [organization_domain](variables.tf#L79) | Organization domain. | string | ✓ | | +| [prefix](variables.tf#L84) | Unique prefix used for resource names. | string | ✓ | | +| [composer_config](variables.tf#L22) | | object({…}) | | {…} | +| [data_force_destroy](variables.tf#L35) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | +| [groups](variables.tf#L46) | Groups. | map(string) | | {…} | +| [network_config](variables.tf#L56) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_services](variables.tf#L89) | List of core services enabled on all projects. | list(string) | | […] | +| [region](variables.tf#L100) | Region used for regional resources. | string | | "europe-west1" | ## Outputs diff --git a/examples/data-solutions/data-platform-foundations/main.tf b/examples/data-solutions/data-platform-foundations/main.tf index f1ab5351..e4a0d203 100644 --- a/examples/data-solutions/data-platform-foundations/main.tf +++ b/examples/data-solutions/data-platform-foundations/main.tf @@ -15,41 +15,13 @@ # tfdoc:file:description Core locals. locals { - _networks = { - load = { - network_name = coalesce(local._shared_vpc_network, module.lod-vpc[0].name) - network = coalesce(var.network_config.network_self_link, module.lod-vpc[0].self_link) - subnet = try(var.network_config.subnet_self_links.load, module.lod-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_lod}-subnet"]) - } - orchestration = { - network_name = coalesce(local._shared_vpc_network, module.orc-vpc[0].name) - network = coalesce(var.network_config.network_self_link, module.orc-vpc[0].self_link) - subnet = try(var.network_config.subnet_self_links.orchestration, module.orc-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_orc}-subnet"]) - } - transformation = { - network_name = coalesce(local._shared_vpc_network, module.trf-vpc[0].name) - network = coalesce(var.network_config.network_self_link, module.trf-vpc[0].self_link) - subnet = try(var.network_config.subnet_self_links.transformation, module.trf-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_trf}-subnet"]) - } + groups = { + for k, v in var.groups : k => "${v}@${var.organization_domain}" + } + groups_iam = { + for k, v in local.groups : k => "group:${v}" } - _shared_vpc_network = try(regex("[a-z]([-a-z0-9]*[a-z0-9])?$", var.network_config.network_self_link), null) - _shared_vpc_project = try(regex("projects/([a-z0-9-]{6,30})", var.network_config.network_self_link)[0], null) - _shared_vpc_service_config = var.network_config.network_self_link != null ? { - attach = true - host_project = local._shared_vpc_project - } : null - - groups = { for k, v in var.groups : k => "${v}@${var.organization.domain}" } - groups_iam = { for k, v in local.groups : k => "group:${v}" } service_encryption_keys = var.service_encryption_keys - - # To create KMS keys in the common projet: uncomment assignement below and comment assignement above - - # service_encryption_keys = { - # bq = module.sec-kms-1.key_ids.bq - # composer = module.sec-kms-2.key_ids.composer - # dataflow = module.sec-kms-2.key_ids.dataflow - # storage = module.sec-kms-1.key_ids.storage - # pubsub = module.sec-kms-0.key_ids.pubsub - # } + shared_vpc_project = try(var.network_config.host_project, null) + use_shared_vpc = var.network_config != null } diff --git a/examples/data-solutions/data-platform-foundations/outputs.tf b/examples/data-solutions/data-platform-foundations/outputs.tf index 1b8db1a5..e5a2de3e 100644 --- a/examples/data-solutions/data-platform-foundations/outputs.tf +++ b/examples/data-solutions/data-platform-foundations/outputs.tf @@ -17,25 +17,25 @@ output "bigquery-datasets" { description = "BigQuery datasets." value = { - lnd-bq-0 = module.lnd-bq-0.dataset_id, - dtl-0-bq-0 = module.dtl-0-bq-0.dataset_id, - dtl-1-bq-0 = module.dtl-1-bq-0.dataset_id, - dtl-2-bq-0 = module.dtl-2-bq-0.dataset_id, - dtl-plg-bq-0 = module.dtl-plg-bq-0.dataset_id, + land-bq-0 = module.land-bq-0.dataset_id, + lake-0-bq-0 = module.lake-0-bq-0.dataset_id, + lake-1-bq-0 = module.lake-1-bq-0.dataset_id, + lake-2-bq-0 = module.lake-2-bq-0.dataset_id, + lake-plg-bq-0 = module.lake-plg-bq-0.dataset_id, } } output "gcs-buckets" { description = "GCS buckets." value = { - dtl-0-cs-0 = module.dtl-0-cs-0.name, - dtl-1-cs-0 = module.dtl-1-cs-0.name, - dtl-2-cs-0 = module.dtl-2-cs-0.name, - dtl-plg-cs-0 = module.dtl-plg-cs-0.name, - lnd-cs-0 = module.lnd-cs-0.name, - lod-cs-df = module.lod-cs-df-0.name, - orc-cs-0 = module.orc-cs-0.name, - trf-cs-df = module.trf-cs-df-0.name, + lake-0-cs-0 = module.lake-0-cs-0.name, + lake-1-cs-0 = module.lake-1-cs-0.name, + lake-2-cs-0 = module.lake-2-cs-0.name, + lake-plg-cs-0 = module.lake-plg-cs-0.name, + land-cs-0 = module.land-cs-0.name, + lod-cs-df = module.load-cs-df-0.name, + orch-cs-0 = module.orch-cs-0.name, + transf-cs-df = module.transf-cs-df-0.name, } } @@ -48,26 +48,26 @@ output "projects" { description = "GCP Projects informations." value = { project_number = { - dtl-0-prj = module.dtl-0-prj.number, - dtl-1-prj = module.dtl-1-prj.number, - dtl-2-prj = module.dtl-2-prj.number, - dtl-plg-prj = module.dtl-plg-prj.number, - exp-prj = module.exp-prj.number, - lnd-prj = module.lnd-prj.number, - lod-prj = module.lod-prj.number, - orc-prj = module.orc-prj.number, - trf-prj = module.trf-prj.number, + lake-0 = module.lake-0-project.number, + lake-1 = module.lake-1-project.number, + lake-2 = module.lake-2-project.number, + lake-plg = module.lake-plg-project.number, + exposure = module.exp-project.number, + landing = module.land-project.number, + load = module.load-project.number, + orchestration = module.orch-project.number, + transformation = module.transf-project.number, } project_id = { - dtl-0-prj = module.dtl-0-prj.project_id, - dtl-1-prj = module.dtl-1-prj.project_id, - dtl-2-prj = module.dtl-2-prj.project_id, - dtl-plg-prj = module.dtl-plg-prj.project_id, - exp-prj = module.exp-prj.project_id, - lnd-prj = module.lnd-prj.project_id, - lod-prj = module.lod-prj.project_id, - orc-prj = module.orc-prj.project_id, - trf-prj = module.trf-prj.project_id, + lake-0 = module.lake-0-project.project_id, + lake-1 = module.lake-1-project.project_id, + lake-2 = module.lake-2-project.project_id, + lake-plg = module.lake-plg-project.project_id, + exposure = module.exp-project.project_id, + landing = module.land-project.project_id, + load = module.load-project.project_id, + orchestration = module.orch-project.project_id, + transformation = module.transf-project.project_id, } } } @@ -75,30 +75,30 @@ output "projects" { output "vpc_network" { description = "VPC network." value = { - lod-vpc = local._networks.load.network - orc-vpc = local._networks.orchestration.network - trf-vpc = local._networks.transformation.network + load = local.load_vpc + orchestration = local.orch_vpc + transformation = local.transf_vpc } } output "vpc_subnet" { description = "VPC subnetworks." value = { - lod-vpc = local._networks.load.subnet - orc-vpc = local._networks.orchestration.subnet - trf-vpc = local._networks.transformation.subnet + load = local.load_subnet + orchestration = local.orch_subnet + transformation = local.transf_subnet } } output "demo_commands" { description = "Demo commands." value = { - 01 = "gsutil -i ${module.lnd-sa-cs-0.email} cp demo/data/*.csv gs://${module.lnd-cs-0.name}" - 02 = "gsutil -i ${module.orc-sa-cmp-0.email} cp demo/data/*.j* gs://${module.orc-cs-0.name}" - 03 = "gsutil -i ${module.orc-sa-cmp-0.email} cp demo/*.py ${google_composer_environment.orc-cmp-0.config[0].dag_gcs_prefix}/" - 04 = "Open ${google_composer_environment.orc-cmp-0.config.0.airflow_uri} and run uploaded DAG." + 01 = "gsutil -i ${module.land-sa-cs-0.email} cp demo/data/*.csv gs://${module.land-cs-0.name}" + 02 = "gsutil -i ${module.orch-sa-cmp-0.email} cp demo/data/*.j* gs://${module.orch-cs-0.name}" + 03 = "gsutil -i ${module.orch-sa-cmp-0.email} cp demo/*.py ${google_composer_environment.orch-cmp-0.config[0].dag_gcs_prefix}/" + 04 = "Open ${google_composer_environment.orch-cmp-0.config.0.airflow_uri} and run uploaded DAG." 05 = <