zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mandatory project creation, refactor

This commit is contained in:
Ludovico Magnocavallo 2022-02-09 17:01:25 +01:00
parent a64e7a8e41
commit 4f4a9cd7ac
17 changed files with 684 additions and 940 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
examples/data-solutions/data-platform-foundations/01-landing.tf
View File

@ -12,67 +12,40 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# tfdoc:file:description Landing project. # tfdoc:file:description land project and resources.
locals { locals {
group_iam_lnd = { land_orch_service_accounts = [
"${local.groups.data-engineers}" = [ module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
]
}
module "land-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "lnd"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor", "roles/bigquery.dataEditor",
"roles/pubsub.editor", "roles/pubsub.editor",
"roles/storage.admin", "roles/storage.admin",
"roles/storage.objectViewer", "roles/storage.objectViewer",
] ]
} }
iam_lnd = { iam = {
"roles/bigquery.dataEditor" = [ "roles/bigquery.dataEditor" = [module.land-sa-bq-0.iam_email]
module.lnd-sa-bq-0.iam_email, "roles/bigquery.dataViewer" = local.land_orch_service_accounts
] "roles/bigquery.jobUser" = [module.orch-sa-cmp-0.iam_email]
"roles/bigquery.dataViewer" = [ "roles/bigquery.user" = [module.load-sa-df-0.iam_email]
module.lod-sa-df-0.iam_email, "roles/pubsub.publisher" = [module.land-sa-ps-0.iam_email]
module.orc-sa-cmp-0.iam_email, "roles/pubsub.subscriber" = local.land_orch_service_accounts
] "roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
"roles/bigquery.jobUser" = [ "roles/storage.objectCreator" = [module.land-sa-cs-0.iam_email]
module.orc-sa-cmp-0.iam_email "roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
] "roles/storage.admin" = [module.load-sa-df-0.iam_email]
"roles/bigquery.user" = [
module.lod-sa-df-0.iam_email
]
"roles/pubsub.publisher" = [
module.lnd-sa-ps-0.iam_email
]
"roles/pubsub.subscriber" = [
module.lod-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email
]
"roles/storage.objectAdmin" = [
module.lod-sa-df-0.iam_email,
]
"roles/storage.objectCreator" = [
module.lnd-sa-cs-0.iam_email,
]
"roles/storage.objectViewer" = [
module.orc-sa-cmp-0.iam_email,
]
"roles/storage.admin" = [
module.lod-sa-df-0.iam_email,
]
} }
prefix_lnd = "${var.prefix}-lnd"
}
# Project
module "lnd-prj" {
source = "../../../modules/project"
name = try(var.project_ids["landing"], "lnd")
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = can(var.project_ids["landing"])
prefix = can(var.project_ids["landing"]) ? var.prefix : null
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_lnd : {}
iam_additive = var.project_create == null ? local.iam_lnd : {}
group_iam = local.group_iam_lnd
services = concat(var.project_services, [ services = concat(var.project_services, [
"bigquery.googleapis.com", "bigquery.googleapis.com",
"bigqueryreservation.googleapis.com", "bigqueryreservation.googleapis.com",
@ -88,3 +61,73 @@ module "lnd-prj" {
storage = [try(local.service_encryption_keys.storage, null)] storage = [try(local.service_encryption_keys.storage, null)]
} }
} }
# Cloud Storage
module "land-sa-cs-0" {
source = "../../../modules/iam-service-account"
project_id = module.land-project.project_id
prefix = var.prefix
name = "lnd-cs-0"
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
]
}
}
module "land-cs-0" {
source = "../../../modules/gcs"
project_id = module.land-project.project_id
prefix = var.prefix
name = "lnd-cs-0"
location = var.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
# retention_policy = {
# retention_period = 7776000 # 90 * 24 * 60 * 60
# is_locked = false
# }
}
# PubSub
module "land-sa-ps-0" {
source = "../../../modules/iam-service-account"
project_id = module.land-project.project_id
prefix = var.prefix
name = "lnd-ps-0"
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
]
}
}
module "land-ps-0" {
source = "../../../modules/pubsub"
project_id = module.land-project.project_id
name = "${var.prefix}-lnd-ps-0"
kms_key = try(local.service_encryption_keys.pubsub, null)
}
# BigQuery
module "land-sa-bq-0" {
source = "../../../modules/iam-service-account"
project_id = module.land-project.project_id
prefix = var.prefix
name = "lnd-bq-0"
iam = {
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
}
}
module "land-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.land-project.project_id
id = "${replace(var.prefix, "-", "_")}lnd_bq_0"
location = var.region
encryption_key = try(local.service_encryption_keys.bq, null)
}

91
examples/data-solutions/data-platform-foundations/01-storage-services.tf
View File

@ -1,91 +0,0 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# tfdoc:file:description Landing storage resources (Bigquery, Cloud PubSub, Cloud Storage).
locals {
lnd_bucket_retention_policy = {
retention_period = 7776000 # 90 * 24 * 60 * 60
is_locked = false
}
}
# Cloud Storage resources
module "lnd-sa-cs-0" {
source = "../../../modules/iam-service-account"
project_id = module.lnd-prj.project_id
name = "cs-0"
prefix = local.prefix_lnd
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
]
}
}
module "lnd-cs-0" {
source = "../../../modules/gcs"
project_id = module.lnd-prj.project_id
name = "cs-0"
prefix = local.prefix_lnd
location = var.location_config.region
storage_class = "REGIONAL"
# retention_policy = local.lnd_bucket_retention_policy
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
# Cloud PubSub resources
module "lnd-sa-ps-0" {
source = "../../../modules/iam-service-account"
project_id = module.lnd-prj.project_id
name = "ps-0"
prefix = local.prefix_lnd
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
]
}
}
module "lnd-ps-0" {
source = "../../../modules/pubsub"
project_id = module.lnd-prj.project_id
name = "${local.prefix_lnd}-ps-0"
kms_key = try(local.service_encryption_keys.pubsub, null)
}
# Bigquery resources
module "lnd-sa-bq-0" {
source = "../../../modules/iam-service-account"
project_id = module.lnd-prj.project_id
name = "bq-0"
prefix = local.prefix_lnd
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
]
}
}
module "lnd-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.lnd-prj.project_id
id = "${replace(local.prefix_lnd, "-", "_")}_bq_0"
location = var.location_config.region
encryption_key = try(local.service_encryption_keys.bq, null)
}

139
examples/data-solutions/data-platform-foundations/02-load.tf
View File

@ -15,52 +15,53 @@
# tfdoc:file:description Load project and VPC. # tfdoc:file:description Load project and VPC.
locals { locals {
group_iam_lod = { load_service_accounts = [
"${local.groups.data-engineers}" = [ "serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
module.load-sa-df-0.iam_email
]
load_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.load-vpc.0.subnet_self_links)[0]
)
load_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.load-vpc.0.self_link
)
}
# Project
module "load-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "lod"
group_iam = {
(local.groups.data-engineers) = [
"roles/compute.viewer", "roles/compute.viewer",
"roles/dataflow.admin", "roles/dataflow.admin",
"roles/dataflow.developer", "roles/dataflow.developer",
"roles/viewer", "roles/viewer",
] ]
} }
iam_lod = { iam = {
"roles/bigquery.jobUser" = [ "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
module.lod-sa-df-0.iam_email
]
"roles/compute.serviceAgent" = [
"serviceAccount:${module.lod-prj.service_accounts.robots.compute}"
]
"roles/dataflow.admin" = [ "roles/dataflow.admin" = [
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
module.lod-sa-df-0.iam_email
]
"roles/dataflow.worker" = [
module.lod-sa-df-0.iam_email
]
"roles/dataflow.serviceAgent" = [
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}"
]
"roles/storage.objectAdmin" = [
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
module.lod-sa-df-0.iam_email
] ]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
# TODO: these are needed on the shared VPC?
# "roles/compute.serviceAgent" = [
# "serviceAccount:${module.load-project.service_accounts.robots.compute}"
# ]
# "roles/dataflow.serviceAgent" = [
# "serviceAccount:${module.load-project.service_accounts.robots.dataflow}"
# ]
} }
prefix_lod = "${var.prefix}-lod"
}
# Project
module "lod-prj" {
source = "../../../modules/project"
name = try(var.project_ids["load"], "lod")
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = can(var.project_ids["load"])
prefix = can(var.project_ids["load"]) ? var.prefix : null
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_lod : {}
iam_additive = var.project_create == null ? local.iam_lod : {}
# group_iam = local.group_iam_lod
services = concat(var.project_services, [ services = concat(var.project_services, [
"bigquery.googleapis.com", "bigquery.googleapis.com",
"bigqueryreservation.googleapis.com", "bigqueryreservation.googleapis.com",
@ -79,37 +80,67 @@ module "lod-prj" {
dataflow = [try(local.service_encryption_keys.dataflow, null)] dataflow = [try(local.service_encryption_keys.dataflow, null)]
storage = [try(local.service_encryption_keys.storage, null)] storage = [try(local.service_encryption_keys.storage, null)]
} }
shared_vpc_service_config = local._shared_vpc_service_config shared_vpc_service_config = local.shared_vpc_project == null ? null : {
attach = true
host_project = local.shared_vpc_project
service_identity_iam = {
# TODO: worker service account
"compute.networkUser" = ["dataflow"]
}
}
} }
module "lod-vpc" { module "load-sa-df-0" {
count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/iam-service-account"
project_id = module.load-project.project_id
prefix = var.prefix
name = "load-df-0"
iam = {
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
"roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
}
}
module "load-cs-df-0" {
source = "../../../modules/gcs"
project_id = module.load-project.project_id
prefix = var.prefix
name = "load-cs-0"
storage_class = "REGIONAL"
location = var.region
encryption_key = try(local.service_encryption_keys.storage, null)
}
# internal VPC resources
module "load-vpc" {
source = "../../../modules/net-vpc" source = "../../../modules/net-vpc"
project_id = module.lod-prj.project_id count = local.use_shared_vpc ? 0 : 1
name = "${local.prefix_lod}-vpc" project_id = module.load-project.project_id
name = "{var.prefix}-default"
subnets = [ subnets = [
{ {
ip_cidr_range = "10.10.0.0/24" ip_cidr_range = "10.10.0.0/24"
name = "${local.prefix_lod}-subnet" name = "default"
region = var.location_config.region region = var.region
secondary_ip_range = {} secondary_ip_range = {}
} }
] ]
} }
module "lod-vpc-firewall" { module "load-vpc-firewall" {
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc-firewall" source = "../../../modules/net-vpc-firewall"
project_id = module.lod-prj.project_id count = local.use_shared_vpc ? 0 : 1
network = local._networks.load.network_name project_id = module.load-project.project_id
network = module.load-vpc.0.name
admin_ranges = ["10.10.0.0/24"] admin_ranges = ["10.10.0.0/24"]
} }
module "lod-nat" { module "load-nat" {
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-cloudnat" source = "../../../modules/net-cloudnat"
project_id = module.lod-prj.project_id count = local.use_shared_vpc ? 0 : 1
region = var.location_config.region project_id = module.load-project.project_id
name = "${local.prefix_lod}-default" name = "${var.prefix}-default"
router_network = module.lod-vpc[0].name region = var.region
router_network = module.load-vpc.0.name
} }

42
examples/data-solutions/data-platform-foundations/02-storage-services.tf
View File

@ -1,42 +0,0 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# tfdoc:file:description Load storage resources (Cloud Storage).
# Cloud Storage
module "lod-sa-df-0" {
source = "../../../modules/iam-service-account"
project_id = module.lod-prj.project_id
name = "df-0"
prefix = local.prefix_lod
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
],
"roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email,
]
}
}
module "lod-cs-df-0" {
source = "../../../modules/gcs"
project_id = module.lod-prj.project_id
name = "cs-0"
prefix = local.prefix_lod
storage_class = "REGIONAL"
location = var.location_config.region
encryption_key = try(local.service_encryption_keys.storage, null)
}

122
examples/data-solutions/data-platform-foundations/03-composer.tf
View File

@ -14,86 +14,96 @@
# tfdoc:file:description Orchestration Cloud Composer definition. # tfdoc:file:description Orchestration Cloud Composer definition.
module "orc-sa-cmp-0" { module "orch-sa-cmp-0" {
source = "../../../modules/iam-service-account" source = "../../../modules/iam-service-account"
project_id = module.orc-prj.project_id project_id = module.orch-project.project_id
name = "cmp-0" prefix = var.prefix
prefix = local.prefix_orc name = "orc-cmp-0"
iam = { iam = {
"roles/iam.serviceAccountTokenCreator" = [ "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
local.groups_iam.data-engineers "roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
],
"roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email,
]
} }
} }
resource "google_composer_environment" "orc-cmp-0" { resource "google_composer_environment" "orch-cmp-0" {
name = "${local.prefix_orc}-cmp-0"
region = var.location_config.region
provider = google-beta provider = google-beta
project = module.orc-prj.project_id project = module.orch-project.project_id
name = "${var.prefix}-orc-cmp-0"
region = var.region
config { config {
node_count = var.composer_config.node_count node_count = var.composer_config.node_count
node_config { node_config {
zone = "${var.location_config.region}-b" zone = "${var.region}-b"
service_account = module.orc-sa-cmp-0.email service_account = module.orch-sa-cmp-0.email
network = local._networks.orchestration.network network = local.orch_vpc
subnetwork = local._networks.orchestration.subnet subnetwork = local.orch_subnet
tags = ["composer-worker", "http-server", "https-server"] tags = ["composer-worker", "http-server", "https-server"]
ip_allocation_policy { ip_allocation_policy {
use_ip_aliases = "true" use_ip_aliases = "true"
cluster_secondary_range_name = try(var.network_config.composer_secondary_ranges.pods, "pods") cluster_secondary_range_name = try(
services_secondary_range_name = try(var.network_config.composer_secondary_ranges.services, "services") var.network_config.composer_secondary_ranges.pods, "pods"
)
services_secondary_range_name = try(
var.network_config.composer_secondary_ranges.services, "services"
)
} }
} }
software_config { software_config {
image_version = var.composer_config.airflow_version image_version = var.composer_config.airflow_version
env_variables = merge( env_variables = merge(
var.composer_config.env_variables, { var.composer_config.env_variables, {
DTL_L0_PRJ = module.dtl-0-prj.project_id DTL_L0_PRJ = module.lake-0-project.project_id
DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id DTL_L0_BQ_DATASET = module.lake-0-bq-0.dataset_id
DTL_L0_GCS = module.dtl-0-cs-0.url DTL_L0_GCS = module.lake-0-cs-0.url
DTL_L1_PRJ = module.dtl-1-prj.project_id DTL_L1_PRJ = module.lake-1-project.project_id
DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id DTL_L1_BQ_DATASET = module.lake-1-bq-0.dataset_id
DTL_L1_GCS = module.dtl-1-cs-0.url DTL_L1_GCS = module.lake-1-cs-0.url
DTL_L2_PRJ = module.dtl-2-prj.project_id DTL_L2_PRJ = module.lake-2-project.project_id
DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id DTL_L2_BQ_DATASET = module.lake-2-bq-0.dataset_id
DTL_L2_GCS = module.dtl-2-cs-0.url DTL_L2_GCS = module.lake-2-cs-0.url
DTL_PLG_PRJ = module.dtl-plg-prj.project_id DTL_PLG_PRJ = module.lake-plg-project.project_id
DTL_PLG_BQ_DATASET = module.dtl-plg-bq-0.dataset_id DTL_PLG_BQ_DATASET = module.lake-plg-bq-0.dataset_id
DTL_PLG_GCS = module.dtl-plg-cs-0.url DTL_PLG_GCS = module.lake-plg-cs-0.url
GCP_REGION = var.location_config.region GCP_REGION = var.region
LND_PRJ = module.lnd-prj.project_id LND_PRJ = module.land-project.project_id
LND_BQ = module.lnd-bq-0.dataset_id LND_BQ = module.land-bq-0.dataset_id
LND_GCS = module.lnd-cs-0.url LND_GCS = module.land-cs-0.url
LND_PS = module.lnd-ps-0.id LND_PS = module.land-ps-0.id
LOD_PRJ = module.lod-prj.project_id LOD_PRJ = module.load-project.project_id
LOD_GCS_STAGING = module.lod-cs-df-0.url LOD_GCS_STAGING = module.load-cs-df-0.url
LOD_NET_VPC = local._networks.load.network LOD_NET_VPC = local.load_vpc
LOD_NET_SUBNET = local._networks.load.subnet LOD_NET_SUBNET = local.load_subnet
LOD_SA_DF = module.lod-sa-df-0.email LOD_SA_DF = module.load-sa-df-0.email
ORC_PRJ = module.orc-prj.project_id ORC_PRJ = module.orch-project.project_id
ORC_GCS = module.orc-cs-0.url ORC_GCS = module.orch-cs-0.url
TRF_PRJ = module.trf-prj.project_id TRF_PRJ = module.transf-project.project_id
TRF_GCS_STAGING = module.trf-cs-df-0.url TRF_GCS_STAGING = module.transf-cs-df-0.url
TRF_NET_VPC = local._networks.transformation.network TRF_NET_VPC = local.transf_vpc
TRF_NET_SUBNET = local._networks.transformation.subnet TRF_NET_SUBNET = local.transf_subnet
TRF_SA_DF = module.trf-sa-df-0.email TRF_SA_DF = module.transf-sa-df-0.email
TRF_SA_BQ = module.trf-sa-bq-0.email TRF_SA_BQ = module.transf-sa-bq-0.email
} }
) )
} }
private_environment_config { private_environment_config {
enable_private_endpoint = "true" enable_private_endpoint = "true"
cloud_sql_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.cloudsql, "10.20.10.0/24") cloud_sql_ipv4_cidr_block = try(
master_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28") var.network_config.composer_ip_ranges.cloudsql, "10.20.10.0/24"
web_server_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28") )
master_ipv4_cidr_block = try(
var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28"
)
web_server_ipv4_cidr_block = try(
var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28"
)
} }
dynamic "encryption_config" { dynamic "encryption_config" {
for_each = try(local.service_encryption_keys.composer != null, false) ? { 1 = 1 } : {} for_each = (
try(local.service_encryption_keys.composer != null, false)
? { 1 = 1 }
: {}
)
content { content {
kms_key_name = try(local.service_encryption_keys.composer, null) kms_key_name = try(local.service_encryption_keys.composer, null)
} }

206
examples/data-solutions/data-platform-foundations/03-orchestration.tf
View File

@ -15,8 +15,26 @@
# tfdoc:file:description Orchestration project and VPC. # tfdoc:file:description Orchestration project and VPC.
locals { locals {
group_iam_orc = { orch_subnet = (
"${local.groups.data-engineers}" = [ local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)
}
module "orch-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "orc"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor", "roles/bigquery.dataEditor",
"roles/bigquery.jobUser", "roles/bigquery.jobUser",
"roles/cloudbuild.builds.editor", "roles/cloudbuild.builds.editor",
@ -30,98 +48,99 @@ locals {
"roles/compute.networkUser" "roles/compute.networkUser"
] ]
} }
iam = {
iam_orc = {
"roles/bigquery.dataEditor" = [ "roles/bigquery.dataEditor" = [
module.lod-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email,
] ]
"roles/bigquery.jobUser" = [ "roles/bigquery.jobUser" = [
module.lod-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email,
] ]
"roles/composer.worker" = [ "roles/composer.worker" = [
module.orc-sa-cmp-0.iam_email module.orch-sa-cmp-0.iam_email
]
"roles/compute.networkUser" = [
module.orc-sa-cmp-0.iam_email,
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}",
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}",
"serviceAccount:${module.orc-prj.service_accounts.cloud_services}"
] ]
"roles/iam.serviceAccountUser" = [ "roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email
] ]
"roles/storage.objectAdmin" = [ "roles/storage.objectAdmin" = [
module.lod-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email,
"serviceAccount:${module.orc-prj.service_accounts.robots.composer}", "serviceAccount:${module.orch-project.service_accounts.robots.composer}",
] ]
"roles/storage.admin" = [ "roles/storage.admin" = [
module.lod-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email module.transf-sa-df-0.iam_email
] ]
} }
oslogin = false
prefix_orc = "${var.prefix}-orc"
}
module "orc-prj" {
source = "../../../modules/project"
name = try(var.project_ids["orchestration"], "orc")
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = can(var.project_ids["orchestration"])
prefix = can(var.project_ids["orchestration"]) ? var.prefix : null
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_orc : {}
iam_additive = var.project_create == null ? local.iam_orc : {}
group_iam = local.group_iam_orc
oslogin = false
policy_boolean = { policy_boolean = {
"constraints/compute.requireOsLogin" = false "constraints/compute.requireOsLogin" = false
} }
services = concat( services = concat(var.project_services, [
var.project_services, "artifactregistry.googleapis.com",
[ "bigquery.googleapis.com",
"artifactregistry.googleapis.com", "bigqueryreservation.googleapis.com",
"bigquery.googleapis.com", "bigquerystorage.googleapis.com",
"bigqueryreservation.googleapis.com", "cloudbuild.googleapis.com",
"bigquerystorage.googleapis.com", "cloudkms.googleapis.com",
"cloudbuild.googleapis.com", "composer.googleapis.com",
"cloudkms.googleapis.com", "compute.googleapis.com",
"composer.googleapis.com", "container.googleapis.com",
"compute.googleapis.com", "containerregistry.googleapis.com",
"container.googleapis.com", "dataflow.googleapis.com",
"containerregistry.googleapis.com", "pubsub.googleapis.com",
"dataflow.googleapis.com", "servicenetworking.googleapis.com",
"pubsub.googleapis.com", "storage.googleapis.com",
"servicenetworking.googleapis.com", "storage-component.googleapis.com"
"storage.googleapis.com",
"storage-component.googleapis.com"
]) ])
service_encryption_key_ids = { service_encryption_key_ids = {
composer = [try(local.service_encryption_keys.composer, null)] composer = [try(local.service_encryption_keys.composer, null)]
storage = [try(local.service_encryption_keys.storage, null)] storage = [try(local.service_encryption_keys.storage, null)]
} }
shared_vpc_service_config = local._shared_vpc_service_config shared_vpc_service_config = local.shared_vpc_project == null ? null : {
attach = true
host_project = local.shared_vpc_project
service_identity_iam = {
"roles/composer.sharedVpcAgent" = [
"composer"
]
"roles/compute.networkUser" = [
"cloudservices", "container-engine", "dataflow"
]
"roles/container.hostServiceAgentUser" = [
"container-engine"
]
}
}
} }
module "orc-vpc" { # Cloud Storage
count = var.network_config.network_self_link != null ? 0 : 1
module "orch-cs-0" {
source = "../../../modules/gcs"
project_id = module.orch-project.project_id
prefix = var.prefix
name = "orc-cs-0"
location = var.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
}
# internal VPC resources
module "orch-vpc" {
source = "../../../modules/net-vpc" source = "../../../modules/net-vpc"
project_id = module.orc-prj.project_id count = local.use_shared_vpc ? 0 : 1
name = "${local.prefix_orc}-vpc" project_id = module.orch-project.project_id
name = "${var.prefix}-default"
subnets = [ subnets = [
{ {
ip_cidr_range = "10.10.0.0/24" ip_cidr_range = "10.10.0.0/24"
name = "${local.prefix_orc}-subnet" name = "default"
region = var.location_config.region region = var.region
secondary_ip_range = { secondary_ip_range = {
pods = "10.10.8.0/22" pods = "10.10.8.0/22"
services = "10.10.12.0/24" services = "10.10.12.0/24"
@ -130,52 +149,19 @@ module "orc-vpc" {
] ]
} }
resource "google_project_iam_binding" "composer_shared_vpc_agent" { module "orch-vpc-firewall" {
count = var.network_config.network_self_link != null ? 1 : 0
project = local._shared_vpc_project
role = "roles/composer.sharedVpcAgent"
members = [
"serviceAccount:${module.orc-prj.service_accounts.robots.composer}"
]
}
resource "google_project_iam_binding" "gke_host_service_agent_user" {
count = var.network_config.network_self_link != null ? 1 : 0
project = local._shared_vpc_project
role = "roles/container.hostServiceAgentUser"
members = [
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}"
]
}
resource "google_project_iam_binding" "composer_network_user_agent" {
count = var.network_config.network_self_link != null ? 1 : 0
project = local._shared_vpc_project
role = "roles/compute.networkUser"
members = [
module.orc-sa-cmp-0.iam_email,
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
"serviceAccount:${module.orc-prj.service_accounts.cloud_services}",
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}",
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}",
]
}
module "orc-vpc-firewall" {
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc-firewall" source = "../../../modules/net-vpc-firewall"
project_id = module.orc-prj.project_id count = local.use_shared_vpc ? 0 : 1
network = local._networks.orchestration.network_name project_id = module.orch-project.project_id
network = module.orch-vpc.0.name
admin_ranges = ["10.10.0.0/24"] admin_ranges = ["10.10.0.0/24"]
} }
module "orc-nat" { module "orch-nat" {
count = var.network_config.network_self_link != null ? 0 : 1 count = local.use_shared_vpc ? 0 : 1
source = "../../../modules/net-cloudnat" source = "../../../modules/net-cloudnat"
project_id = module.orc-prj.project_id project_id = module.orch-project.project_id
region = var.location_config.region name = "${var.prefix}-default"
name = "${local.prefix_orc}-default" region = var.region
router_network = local._networks.orchestration.network_name router_network = module.orch-vpc.0.name
} }

27
examples/data-solutions/data-platform-foundations/03-storage-services.tf
View File

@ -1,27 +0,0 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# tfdoc:file:description Orchestration storage resources (Cloud Storage).
# Cloud Storage
module "orc-cs-0" {
source = "../../../modules/gcs"
project_id = module.orc-prj.project_id
name = "cs-0"
prefix = local.prefix_orc
location = var.location_config.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
}

61
examples/data-solutions/data-platform-foundations/04-storage-services.tf
View File

@ -1,61 +0,0 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# tfdoc:file:description Landing storage resources (Bigquery, Cloud Storage).
# Cloud Storage
module "trf-sa-df-0" {
source = "../../../modules/iam-service-account"
project_id = module.trf-prj.project_id
name = "df-0"
prefix = local.prefix_trf
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
module.orc-sa-cmp-0.iam_email,
],
"roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email,
]
}
}
module "trf-cs-df-0" {
source = "../../../modules/gcs"
project_id = module.trf-prj.project_id
name = "cs-0"
prefix = local.prefix_trf
location = var.location_config.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
}
# Bigquery
module "trf-sa-bq-0" {
source = "../../../modules/iam-service-account"
project_id = module.trf-prj.project_id
name = "bq-0"
prefix = local.prefix_trf
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
module.orc-sa-cmp-0.iam_email,
],
"roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email,
]
}
}

140
examples/data-solutions/data-platform-foundations/04-transformation.tf
View File

@ -15,47 +15,49 @@
# tfdoc:file:description Trasformation project and VPC. # tfdoc:file:description Trasformation project and VPC.
locals { locals {
group_iam_trf = { transf_subnet = (
"${local.groups.data-engineers}" = [ local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.transf-vpc.0.subnet_self_links)[0]
)
transf_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.transf-vpc.0.self_link
)
}
module "transf-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "trf"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.jobUser", "roles/bigquery.jobUser",
"roles/dataflow.admin", "roles/dataflow.admin",
] ]
} }
iam_trf = { iam = {
"roles/bigquery.dataViewer" = [ "roles/bigquery.dataViewer" = [
module.orc-sa-cmp-0.iam_email module.orch-sa-cmp-0.iam_email
] ]
"roles/bigquery.jobUser" = [ "roles/bigquery.jobUser" = [
module.trf-sa-bq-0.iam_email, module.transf-sa-bq-0.iam_email,
] ]
"roles/dataflow.admin" = [ "roles/dataflow.admin" = [
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email,
] ]
"roles/dataflow.worker" = [ "roles/dataflow.worker" = [
module.trf-sa-df-0.iam_email module.transf-sa-df-0.iam_email
] ]
"roles/storage.objectAdmin" = [ "roles/storage.objectAdmin" = [
module.trf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email,
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}" "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
] ]
} }
prefix_trf = "${var.prefix}-trf"
}
# Project
module "trf-prj" {
source = "../../../modules/project"
name = try(var.project_ids["trasformation"], "trf")
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = can(var.project_ids["trasformation"])
prefix = can(var.project_ids["trasformation"]) ? var.prefix : null
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_trf : {}
iam_additive = var.project_create == null ? local.iam_trf : {}
group_iam = local.group_iam_trf
services = concat(var.project_services, [ services = concat(var.project_services, [
"bigquery.googleapis.com", "bigquery.googleapis.com",
"bigqueryreservation.googleapis.com", "bigqueryreservation.googleapis.com",
@ -73,37 +75,89 @@ module "trf-prj" {
dataflow = [try(local.service_encryption_keys.dataflow, null)] dataflow = [try(local.service_encryption_keys.dataflow, null)]
storage = [try(local.service_encryption_keys.storage, null)] storage = [try(local.service_encryption_keys.storage, null)]
} }
shared_vpc_service_config = local._shared_vpc_service_config shared_vpc_service_config = local.shared_vpc_project == null ? null : {
attach = true
host_project = local.shared_vpc_project
service_identity_iam = {}
}
} }
module "trf-vpc" { # Cloud Storage
count = var.network_config.network_self_link != null ? 0 : 1
module "transf-sa-df-0" {
source = "../../../modules/iam-service-account"
project_id = module.transf-project.project_id
prefix = var.prefix
name = "trf-df-0"
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
module.orch-sa-cmp-0.iam_email
],
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
]
}
}
module "transf-cs-df-0" {
source = "../../../modules/gcs"
project_id = module.transf-project.project_id
prefix = var.prefix
name = "trf-cs-0"
location = var.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
}
# BigQuery
module "transf-sa-bq-0" {
source = "../../../modules/iam-service-account"
project_id = module.transf-project.project_id
prefix = var.prefix
name = "trf-bq-0"
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
module.orch-sa-cmp-0.iam_email
],
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
]
}
}
# internal VPC resources
module "transf-vpc" {
source = "../../../modules/net-vpc" source = "../../../modules/net-vpc"
project_id = module.trf-prj.project_id count = local.use_shared_vpc ? 0 : 1
name = "${local.prefix_trf}-vpc" project_id = module.transf-project.project_id
name = "{var.prefix}-default"
subnets = [ subnets = [
{ {
ip_cidr_range = "10.10.0.0/24" ip_cidr_range = "10.10.0.0/24"
name = "${local.prefix_trf}-subnet" name = "default"
region = var.location_config.region region = var.region
secondary_ip_range = {} secondary_ip_range = {}
} }
] ]
} }
module "trf-vpc-firewall" { module "transf-vpc-firewall" {
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc-firewall" source = "../../../modules/net-vpc-firewall"
project_id = module.trf-prj.project_id count = local.use_shared_vpc ? 0 : 1
network = local._networks.transformation.network_name project_id = module.transf-project.project_id
network = module.transf-vpc.0.name
admin_ranges = ["10.10.0.0/24"] admin_ranges = ["10.10.0.0/24"]
} }
module "trf-nat" { module "transf-nat" {
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-cloudnat" source = "../../../modules/net-cloudnat"
project_id = module.trf-prj.project_id count = local.use_shared_vpc ? 0 : 1
region = var.location_config.region project_id = module.transf-project.project_id
name = "${local.prefix_trf}-default" name = "${var.prefix}-default"
router_network = module.trf-vpc[0].name region = var.region
router_network = module.transf-vpc.0.name
} }

251
examples/data-solutions/data-platform-foundations/05-datalake.tf
View File

@ -15,12 +15,12 @@
# tfdoc:file:description Datalake projects. # tfdoc:file:description Datalake projects.
locals { locals {
group_iam_dtl = { lake_group_iam = {
"${local.groups.data-engineers}" = [ (local.groups.data-engineers) = [
"roles/bigquery.dataEditor", "roles/bigquery.dataEditor",
"roles/storage.admin", "roles/storage.admin",
], ],
"${local.groups.data-analysts}" = [ (local.groups.data-analysts) = [
"roles/bigquery.dataViewer", "roles/bigquery.dataViewer",
"roles/bigquery.jobUser", "roles/bigquery.jobUser",
"roles/bigquery.user", "roles/bigquery.user",
@ -29,150 +29,185 @@ locals {
"roles/storage.objectViewer", "roles/storage.objectViewer",
] ]
} }
iam_dtl = { lake_iam = {
"roles/bigquery.dataEditor" = [ "roles/bigquery.dataEditor" = [
module.lod-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.trf-sa-bq-0.iam_email, module.transf-sa-bq-0.iam_email,
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email,
] ]
"roles/bigquery.jobUser" = [ "roles/bigquery.jobUser" = [
module.lod-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
] ]
"roles/storage.admin" = [ "roles/storage.admin" = [
module.lod-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
] ]
"roles/storage.objectCreator" = [ "roles/storage.objectCreator" = [
module.lod-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.trf-sa-bq-0.iam_email, module.transf-sa-bq-0.iam_email,
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email,
] ]
"roles/storage.objectViewer" = [ "roles/storage.objectViewer" = [
module.trf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.trf-sa-bq-0.iam_email, module.transf-sa-bq-0.iam_email,
module.orc-sa-cmp-0.iam_email, module.orch-sa-cmp-0.iam_email,
] ]
} }
prefix_dtl = "${var.prefix}-dtl" lake_services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
} }
# Project # Project
module "dtl-0-prj" { module "lake-0-project" {
source = "../../../modules/project" source = "../../../modules/project"
name = try(var.project_ids["datalake-l0"], "dtl-0") parent = var.folder_id
parent = try(var.project_create.parent, null) billing_account = var.billing_account_id
billing_account = try(var.project_create.billing_account_id, null) prefix = var.prefix
project_create = can(var.project_ids["datalake-l0"]) name = "dtl-0"
prefix = can(var.project_ids["datalake-l0"]) ? var.prefix : null group_iam = local.lake_group_iam
# additive IAM bindings avoid disrupting bindings in existing project iam = local.lake_iam
iam = var.project_create != null ? local.iam_dtl : {} services = local.lake_services
iam_additive = var.project_create == null ? local.iam_dtl : {}
group_iam = local.group_iam_dtl
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
service_encryption_key_ids = { service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)] bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)] storage = [try(local.service_encryption_keys.storage, null)]
} }
} }
module "dtl-1-prj" { module "lake-1-project" {
source = "../../../modules/project" source = "../../../modules/project"
name = try(var.project_ids["datalake-l1"], "dtl-1") parent = var.folder_id
parent = try(var.project_create.parent, null) billing_account = var.billing_account_id
billing_account = try(var.project_create.billing_account_id, null) prefix = var.prefix
project_create = can(var.project_ids["datalake-l1"]) name = "dtl-1"
prefix = can(var.project_ids["datalake-l1"]) ? var.prefix : null group_iam = local.lake_group_iam
# additive IAM bindings avoid disrupting bindings in existing project iam = local.lake_iam
iam = var.project_create != null ? local.iam_dtl : {} services = local.lake_services
iam_additive = var.project_create == null ? local.iam_dtl : {}
group_iam = local.group_iam_dtl
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
service_encryption_key_ids = { service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)] bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)] storage = [try(local.service_encryption_keys.storage, null)]
} }
} }
module "dtl-2-prj" { module "lake-2-project" {
source = "../../../modules/project" source = "../../../modules/project"
name = try(var.project_ids["datalake-l2"], "dtl-2") parent = var.folder_id
parent = try(var.project_create.parent, null) billing_account = var.billing_account_id
billing_account = try(var.project_create.billing_account_id, null) prefix = var.prefix
project_create = can(var.project_ids["datalake-l2"]) name = "dtl-2"
prefix = can(var.project_ids["datalake-l2"]) ? var.prefix : null group_iam = local.lake_group_iam
# additive IAM bindings avoid disrupting bindings in existing project iam = local.lake_iam
iam = var.project_create != null ? local.iam_dtl : {} services = local.lake_services
iam_additive = var.project_create == null ? local.iam_dtl : {}
group_iam = local.group_iam_dtl
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
service_encryption_key_ids = { service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)] bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)] storage = [try(local.service_encryption_keys.storage, null)]
} }
} }
module "dtl-plg-prj" { module "lake-plg-project" {
source = "../../../modules/project" source = "../../../modules/project"
name = try(var.project_ids["datalake-playground"], "dtl-plg") parent = var.folder_id
parent = try(var.project_create.parent, null) billing_account = var.billing_account_id
billing_account = try(var.project_create.billing_account_id, null) prefix = var.prefix
project_create = can(var.project_ids["datalake-playground"]) name = "dtl-plg"
prefix = can(var.project_ids["datalake-playground"]) ? var.prefix : null group_iam = local.lake_group_iam
# additive IAM bindings avoid disrupting bindings in existing project iam = local.lake_iam
iam = var.project_create != null ? local.iam_dtl : {} services = local.lake_services
iam_additive = var.project_create == null ? local.iam_dtl : {}
group_iam = local.group_iam_dtl
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
service_encryption_key_ids = { service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)] bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)] storage = [try(local.service_encryption_keys.storage, null)]
} }
} }
# Bigquery
module "lake-0-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.lake-0-project.project_id
id = "${replace(var.prefix, "-", "_")}_dtl_0_bq_0"
location = var.region
encryption_key = try(local.service_encryption_keys.bq, null)
}
module "lake-1-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.lake-1-project.project_id
id = "${replace(var.prefix, "-", "_")}_dtl_1_bq_0"
location = var.region
encryption_key = try(local.service_encryption_keys.bq, null)
}
module "lake-2-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.lake-2-project.project_id
id = "${replace(var.prefix, "-", "_")}_dtl_2_bq_0"
location = var.region
encryption_key = try(local.service_encryption_keys.bq, null)
}
module "lake-plg-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.lake-plg-project.project_id
id = "${replace(var.prefix, "-", "_")}_dtl_plg_bq_0"
location = var.region
encryption_key = try(local.service_encryption_keys.bq, null)
}
# Cloud storage
module "lake-0-cs-0" {
source = "../../../modules/gcs"
project_id = module.lake-0-project.project_id
prefix = var.prefix
name = "dtl-0-cs-0"
location = var.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
module "lake-1-cs-0" {
source = "../../../modules/gcs"
project_id = module.lake-1-project.project_id
prefix = var.prefix
name = "dtl-1-cs-0"
location = var.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
module "lake-2-cs-0" {
source = "../../../modules/gcs"
project_id = module.lake-2-project.project_id
prefix = var.prefix
name = "dtl-2-cs-0"
location = var.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
module "lake-plg-cs-0" {
source = "../../../modules/gcs"
project_id = module.lake-plg-project.project_id
prefix = var.prefix
name = "dtl-plg-cs-0"
location = var.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}

95
examples/data-solutions/data-platform-foundations/05-storage-services.tf
View File

@ -1,95 +0,0 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# tfdoc:file:description Datalake storage resources (Bigquery, Cloud Storage).
# Bigquery
module "dtl-0-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dtl-0-prj.project_id
id = "${replace(local.prefix_dtl, "-", "_")}_0_bq_0"
location = var.location_config.region
encryption_key = try(local.service_encryption_keys.bq, null)
}
module "dtl-1-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dtl-1-prj.project_id
id = "${replace(local.prefix_dtl, "-", "_")}_1_bq_0"
location = var.location_config.region
encryption_key = try(local.service_encryption_keys.bq, null)
}
module "dtl-2-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dtl-2-prj.project_id
id = "${replace(local.prefix_dtl, "-", "_")}_2_bq_0"
location = var.location_config.region
encryption_key = try(local.service_encryption_keys.bq, null)
}
module "dtl-plg-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dtl-plg-prj.project_id
id = "${replace(local.prefix_dtl, "-", "_")}_plg_bq_0"
location = var.location_config.region
encryption_key = try(local.service_encryption_keys.bq, null)
}
# Cloud storage
module "dtl-0-cs-0" {
source = "../../../modules/gcs"
project_id = module.dtl-0-prj.project_id
name = "0-cs-0"
prefix = local.prefix_dtl
location = var.location_config.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
module "dtl-1-cs-0" {
source = "../../../modules/gcs"
project_id = module.dtl-1-prj.project_id
name = "1-cs-0"
prefix = local.prefix_dtl
location = var.location_config.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
module "dtl-2-cs-0" {
source = "../../../modules/gcs"
project_id = module.dtl-2-prj.project_id
name = "2-cs-0"
prefix = local.prefix_dtl
location = var.location_config.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
module "dtl-plg-cs-0" {
source = "../../../modules/gcs"
project_id = module.dtl-plg-prj.project_id
name = "plg-cs-0"
prefix = local.prefix_dtl
location = var.location_config.region
storage_class = "REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}

43
examples/data-solutions/data-platform-foundations/06-common.tf
View File

@ -14,39 +14,28 @@
# tfdoc:file:description common project. # tfdoc:file:description common project.
locals { module "common-project" {
group_iam_cmn = { source = "../../../modules/project"
"${local.groups.data-engineers}" = [ parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "cmn"
group_iam = {
(local.groups.data-engineers) = [
"roles/dlp.reader", "roles/dlp.reader",
"roles/dlp.user", "roles/dlp.user",
"roles/dlp.estimatesAdmin", "roles/dlp.estimatesAdmin",
], ]
"${local.groups.data-security}" = [ (local.groups.data-security) = [
"roles/dlp.admin", "roles/dlp.admin",
],
}
iam_cmn = {
"roles/dlp.user" = [
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email
] ]
} }
prefix_cmn = "${var.prefix}-cmn" iam = {
} "roles/dlp.user" = [
module.load-sa-df-0.iam_email,
# Project module.transf-sa-df-0.iam_email
]
module "cmn-prj" { }
source = "../../../modules/project"
name = try(var.project_ids["common"], "cmn")
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = can(var.project_ids["common"])
prefix = can(var.project_ids["common"]) ? var.prefix : null
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_cmn : {}
iam_additive = var.project_create == null ? local.iam_cmn : {}
group_iam = local.group_iam_cmn
services = concat(var.project_services, [ services = concat(var.project_services, [
"datacatalog.googleapis.com", "datacatalog.googleapis.com",
"dlp.googleapis.com", "dlp.googleapis.com",

29
examples/data-solutions/data-platform-foundations/07-exposure.tf
View File

@ -14,29 +14,10 @@
# tfdoc:file:description common project. # tfdoc:file:description common project.
locals { module "exp-project" {
group_iam_exp = {
#TODO add group => role mapping to asign on exposure project
}
iam_exp = {
#TODO add role => service account mapping to assign roles on exposure project
}
prefix_exp = "${var.prefix}-exp"
}
# Project
module "exp-prj" {
source = "../../../modules/project" source = "../../../modules/project"
name = try(var.project_ids["exposure"], "exp") parent = var.folder_id
parent = try(var.project_create.parent, null) billing_account = var.billing_account_id
billing_account = try(var.project_create.billing_account_id, null) prefix = var.prefix
project_create = can(var.project_ids["exposure"]) name = "exp"
prefix = can(var.project_ids["exposure"]) ? var.prefix : null
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_exp : {}
iam_additive = var.project_create == null ? local.iam_exp : {}
group_iam = local.group_iam_exp
services = concat(var.project_services, [
])
} }

20
examples/data-solutions/data-platform-foundations/README.md
View File

@ -241,16 +241,16 @@ Description of commands:
| name | description | type | required | default | | name | description | type | required | default |
|---|---|:---:|:---:|:---:| |---|---|:---:|:---:|:---:|
| [network_config](variables.tf#L47) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10; network_self_link &#61; string&#10; subnet_self_links &#61; object&#40;&#123;&#10; load &#61; string&#10; transformation &#61; string&#10; orchestration &#61; string&#10; &#125;&#41;&#10; composer_ip_ranges &#61; object&#40;&#123;&#10; cloudsql &#61; string&#10; gke_master &#61; string&#10; web_server &#61; string&#10; &#125;&#41;&#10; composer_secondary_ranges &#61; object&#40;&#123;&#10; pods &#61; string&#10; services &#61; string&#10; &#125;&#41;&#10;&#125;&#41;&#10;&#10;&#10;default &#61; &#123;&#10; network_self_link &#61; null&#10; subnet_self_links &#61; null&#10; composer_ip_ranges &#61; null&#10; composer_secondary_ranges &#61; null&#10;&#125;">object&#40;&#123;&#8230;&#125;</code> | ✓ | | | [billing_account_id](variables.tf#L17) | Billing account id. | <code>string</code> | ✓ | |
| [organization](variables.tf#L75) | Organization details. | <code title="object&#40;&#123;&#10; domain &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | | [folder_id](variables.tf#L41) | Folder to be used for the networking resources in folders/nnnn format. | <code>string</code> | ✓ | |
| [prefix](variables.tf#L82) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | <code>string</code> | ✓ | | | [organization_domain](variables.tf#L79) | Organization domain. | <code>string</code> | ✓ | |
| [composer_config](variables.tf#L18) | | <code title="object&#40;&#123;&#10; node_count &#61; number&#10; airflow_version &#61; string&#10; env_variables &#61; map&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; node_count &#61; 3&#10; airflow_version &#61; &#34;composer-1.17.5-airflow-2.1.4&#34;&#10; env_variables &#61; &#123;&#125;&#10;&#125;">&#123;&#8230;&#125;</code> | | [prefix](variables.tf#L84) | Unique prefix used for resource names. | <code>string</code> | ✓ | |
| [data_force_destroy](variables.tf#L31) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> | | [composer_config](variables.tf#L22) | | <code title="object&#40;&#123;&#10; node_count &#61; number&#10; airflow_version &#61; string&#10; env_variables &#61; map&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; node_count &#61; 3&#10; airflow_version &#61; &#34;composer-1.17.5-airflow-2.1.4&#34;&#10; env_variables &#61; &#123;&#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [groups](variables.tf#L37) | Groups. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; data-analysts &#61; &#34;gcp-data-analysts&#34;&#10; data-engineers &#61; &#34;gcp-data-engineers&#34;&#10; data-security &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | | [data_force_destroy](variables.tf#L35) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
| [location_config](variables.tf#L135) | Locations where resources will be deployed. Map to configure region and multiregion specs. | <code title="object&#40;&#123;&#10; region &#61; string&#10; multi_region &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; region &#61; &#34;europe-west1&#34;&#10; multi_region &#61; &#34;eu&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | | [groups](variables.tf#L46) | Groups. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; data-analysts &#61; &#34;gcp-data-analysts&#34;&#10; data-engineers &#61; &#34;gcp-data-engineers&#34;&#10; data-security &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [project_create](variables.tf#L87) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10; billing_account_id &#61; string&#10; parent &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | | [network_config](variables.tf#L56) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10; host_project &#61; string&#10; network_self_link &#61; string&#10; subnet_self_links &#61; object&#40;&#123;&#10; load &#61; string&#10; transformation &#61; string&#10; orchestration &#61; string&#10; &#125;&#41;&#10; composer_ip_ranges &#61; object&#40;&#123;&#10; cloudsql &#61; string&#10; gke_master &#61; string&#10; web_server &#61; string&#10; &#125;&#41;&#10; composer_secondary_ranges &#61; object&#40;&#123;&#10; pods &#61; string&#10; services &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [project_ids](variables.tf#L96) | Project id, references existing project if `project_create` is null. | <code title="object&#40;&#123;&#10; landing &#61; string&#10; load &#61; string&#10; orchestration &#61; string&#10; trasformation &#61; string&#10; datalake-l0 &#61; string&#10; datalake-l1 &#61; string&#10; datalake-l2 &#61; string&#10; datalake-playground &#61; string&#10; common &#61; string&#10; exposure &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; landing &#61; &#34;lnd&#34;&#10; load &#61; &#34;lod&#34;&#10; orchestration &#61; &#34;orc&#34;&#10; trasformation &#61; &#34;trf&#34;&#10; datalake-l0 &#61; &#34;dtl-0&#34;&#10; datalake-l1 &#61; &#34;dtl-1&#34;&#10; datalake-l2 &#61; &#34;dtl-2&#34;&#10; datalake-playground &#61; &#34;dtl-plg&#34;&#10; common &#61; &#34;cmn&#34;&#10; exposure &#61; &#34;exp&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | | [project_services](variables.tf#L89) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> | | <code title="&#91;&#10; &#34;cloudresourcemanager.googleapis.com&#34;,&#10; &#34;iam.googleapis.com&#34;,&#10; &#34;serviceusage.googleapis.com&#34;,&#10; &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |
| [project_services](variables.tf#L124) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> | | <code title="&#91;&#10; &#34;cloudresourcemanager.googleapis.com&#34;,&#10; &#34;iam.googleapis.com&#34;,&#10; &#34;serviceusage.googleapis.com&#34;,&#10; &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> | | [region](variables.tf#L100) | Region used for regional resources. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
## Outputs ## Outputs

42
examples/data-solutions/data-platform-foundations/main.tf
View File

@ -15,41 +15,13 @@
# tfdoc:file:description Core locals. # tfdoc:file:description Core locals.
locals { locals {
_networks = { groups = {
load = { for k, v in var.groups : k => "${v}@${var.organization_domain}"
network_name = coalesce(local._shared_vpc_network, module.lod-vpc[0].name) }
network = coalesce(var.network_config.network_self_link, module.lod-vpc[0].self_link) groups_iam = {
subnet = try(var.network_config.subnet_self_links.load, module.lod-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_lod}-subnet"]) for k, v in local.groups : k => "group:${v}"
}
orchestration = {
network_name = coalesce(local._shared_vpc_network, module.orc-vpc[0].name)
network = coalesce(var.network_config.network_self_link, module.orc-vpc[0].self_link)
subnet = try(var.network_config.subnet_self_links.orchestration, module.orc-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_orc}-subnet"])
}
transformation = {
network_name = coalesce(local._shared_vpc_network, module.trf-vpc[0].name)
network = coalesce(var.network_config.network_self_link, module.trf-vpc[0].self_link)
subnet = try(var.network_config.subnet_self_links.transformation, module.trf-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_trf}-subnet"])
}
} }
_shared_vpc_network = try(regex("[a-z]([-a-z0-9]*[a-z0-9])?$", var.network_config.network_self_link), null)
_shared_vpc_project = try(regex("projects/([a-z0-9-]{6,30})", var.network_config.network_self_link)[0], null)
_shared_vpc_service_config = var.network_config.network_self_link != null ? {
attach = true
host_project = local._shared_vpc_project
} : null
groups = { for k, v in var.groups : k => "${v}@${var.organization.domain}" }
groups_iam = { for k, v in local.groups : k => "group:${v}" }
service_encryption_keys = var.service_encryption_keys service_encryption_keys = var.service_encryption_keys
shared_vpc_project = try(var.network_config.host_project, null)
# To create KMS keys in the common projet: uncomment assignement below and comment assignement above use_shared_vpc = var.network_config != null
# service_encryption_keys = {
# bq = module.sec-kms-1.key_ids.bq
# composer = module.sec-kms-2.key_ids.composer
# dataflow = module.sec-kms-2.key_ids.dataflow
# storage = module.sec-kms-1.key_ids.storage
# pubsub = module.sec-kms-0.key_ids.pubsub
# }
} }

84
examples/data-solutions/data-platform-foundations/outputs.tf
View File

@ -17,25 +17,25 @@
output "bigquery-datasets" { output "bigquery-datasets" {
description = "BigQuery datasets." description = "BigQuery datasets."
value = { value = {
lnd-bq-0 = module.lnd-bq-0.dataset_id, land-bq-0 = module.land-bq-0.dataset_id,
dtl-0-bq-0 = module.dtl-0-bq-0.dataset_id, lake-0-bq-0 = module.lake-0-bq-0.dataset_id,
dtl-1-bq-0 = module.dtl-1-bq-0.dataset_id, lake-1-bq-0 = module.lake-1-bq-0.dataset_id,
dtl-2-bq-0 = module.dtl-2-bq-0.dataset_id, lake-2-bq-0 = module.lake-2-bq-0.dataset_id,
dtl-plg-bq-0 = module.dtl-plg-bq-0.dataset_id, lake-plg-bq-0 = module.lake-plg-bq-0.dataset_id,
} }
} }
output "gcs-buckets" { output "gcs-buckets" {
description = "GCS buckets." description = "GCS buckets."
value = { value = {
dtl-0-cs-0 = module.dtl-0-cs-0.name, lake-0-cs-0 = module.lake-0-cs-0.name,
dtl-1-cs-0 = module.dtl-1-cs-0.name, lake-1-cs-0 = module.lake-1-cs-0.name,
dtl-2-cs-0 = module.dtl-2-cs-0.name, lake-2-cs-0 = module.lake-2-cs-0.name,
dtl-plg-cs-0 = module.dtl-plg-cs-0.name, lake-plg-cs-0 = module.lake-plg-cs-0.name,
lnd-cs-0 = module.lnd-cs-0.name, land-cs-0 = module.land-cs-0.name,
lod-cs-df = module.lod-cs-df-0.name, lod-cs-df = module.load-cs-df-0.name,
orc-cs-0 = module.orc-cs-0.name, orch-cs-0 = module.orch-cs-0.name,
trf-cs-df = module.trf-cs-df-0.name, transf-cs-df = module.transf-cs-df-0.name,
} }
} }
@ -48,26 +48,26 @@ output "projects" {
description = "GCP Projects informations." description = "GCP Projects informations."
value = { value = {
project_number = { project_number = {
dtl-0-prj = module.dtl-0-prj.number, lake-0 = module.lake-0-project.number,
dtl-1-prj = module.dtl-1-prj.number, lake-1 = module.lake-1-project.number,
dtl-2-prj = module.dtl-2-prj.number, lake-2 = module.lake-2-project.number,
dtl-plg-prj = module.dtl-plg-prj.number, lake-plg = module.lake-plg-project.number,
exp-prj = module.exp-prj.number, exposure = module.exp-project.number,
lnd-prj = module.lnd-prj.number, landing = module.land-project.number,
lod-prj = module.lod-prj.number, load = module.load-project.number,
orc-prj = module.orc-prj.number, orchestration = module.orch-project.number,
trf-prj = module.trf-prj.number, transformation = module.transf-project.number,
} }
project_id = { project_id = {
dtl-0-prj = module.dtl-0-prj.project_id, lake-0 = module.lake-0-project.project_id,
dtl-1-prj = module.dtl-1-prj.project_id, lake-1 = module.lake-1-project.project_id,
dtl-2-prj = module.dtl-2-prj.project_id, lake-2 = module.lake-2-project.project_id,
dtl-plg-prj = module.dtl-plg-prj.project_id, lake-plg = module.lake-plg-project.project_id,
exp-prj = module.exp-prj.project_id, exposure = module.exp-project.project_id,
lnd-prj = module.lnd-prj.project_id, landing = module.land-project.project_id,
lod-prj = module.lod-prj.project_id, load = module.load-project.project_id,
orc-prj = module.orc-prj.project_id, orchestration = module.orch-project.project_id,
trf-prj = module.trf-prj.project_id, transformation = module.transf-project.project_id,
} }
} }
} }
@ -75,30 +75,30 @@ output "projects" {
output "vpc_network" { output "vpc_network" {
description = "VPC network." description = "VPC network."
value = { value = {
lod-vpc = local._networks.load.network load = local.load_vpc
orc-vpc = local._networks.orchestration.network orchestration = local.orch_vpc
trf-vpc = local._networks.transformation.network transformation = local.transf_vpc
} }
} }
output "vpc_subnet" { output "vpc_subnet" {
description = "VPC subnetworks." description = "VPC subnetworks."
value = { value = {
lod-vpc = local._networks.load.subnet load = local.load_subnet
orc-vpc = local._networks.orchestration.subnet orchestration = local.orch_subnet
trf-vpc = local._networks.transformation.subnet transformation = local.transf_subnet
} }
} }
output "demo_commands" { output "demo_commands" {
description = "Demo commands." description = "Demo commands."
value = { value = {
01 = "gsutil -i ${module.lnd-sa-cs-0.email} cp demo/data/*.csv gs://${module.lnd-cs-0.name}" 01 = "gsutil -i ${module.land-sa-cs-0.email} cp demo/data/*.csv gs://${module.land-cs-0.name}"
02 = "gsutil -i ${module.orc-sa-cmp-0.email} cp demo/data/*.j* gs://${module.orc-cs-0.name}" 02 = "gsutil -i ${module.orch-sa-cmp-0.email} cp demo/data/*.j* gs://${module.orch-cs-0.name}"
03 = "gsutil -i ${module.orc-sa-cmp-0.email} cp demo/*.py ${google_composer_environment.orc-cmp-0.config[0].dag_gcs_prefix}/" 03 = "gsutil -i ${module.orch-sa-cmp-0.email} cp demo/*.py ${google_composer_environment.orch-cmp-0.config[0].dag_gcs_prefix}/"
04 = "Open ${google_composer_environment.orc-cmp-0.config.0.airflow_uri} and run uploaded DAG." 04 = "Open ${google_composer_environment.orch-cmp-0.config.0.airflow_uri} and run uploaded DAG."
05 = <<EOT 05 = <<EOT
bq query --project_id=${module.dtl-2-prj.project_id} --use_legacy_sql=false 'SELECT * FROM `${module.dtl-2-prj.project_id}.${module.dtl-2-bq-0.dataset_id}.customer_purchase` LIMIT 1000'" bq query --project_id=${module.lake-2-project.project_id} --use_legacy_sql=false 'SELECT * FROM `${module.lake-2-project.project_id}.${module.lake-2-bq-0.dataset_id}.customer_purchase` LIMIT 1000'"
EOT EOT
} }
} }

85
examples/data-solutions/data-platform-foundations/variables.tf
View File

@ -14,6 +14,10 @@
# tfdoc:file:description Terraform Variables. # tfdoc:file:description Terraform Variables.
variable "billing_account_id" {
description = "Billing account id."
type = string
}
variable "composer_config" { variable "composer_config" {
type = object({ type = object({
@ -34,6 +38,11 @@ variable "data_force_destroy" {
default = false default = false
} }
variable "folder_id" {
description = "Folder to be used for the networking resources in folders/nnnn format."
type = string
}
variable "groups" { variable "groups" {
description = "Groups." description = "Groups."
type = map(string) type = map(string)
@ -47,6 +56,7 @@ variable "groups" {
variable "network_config" { variable "network_config" {
description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
type = object({ type = object({
host_project = string
network_self_link = string network_self_link = string
subnet_self_links = object({ subnet_self_links = object({
load = string load = string
@ -63,62 +73,17 @@ variable "network_config" {
services = string services = string
}) })
}) })
default = {
network_self_link = null
subnet_self_links = null
composer_ip_ranges = null
composer_secondary_ranges = null
}
}
variable "organization" {
description = "Organization details."
type = object({
domain = string
})
}
variable "prefix" {
description = "Unique prefix used for resource names. Not used for projects if 'project_create' is null."
type = string
}
variable "project_create" {
description = "Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
type = object({
billing_account_id = string
parent = string
})
default = null default = null
} }
variable "project_ids" { variable "organization_domain" {
description = "Project id, references existing project if `project_create` is null." description = "Organization domain."
type = object({ type = string
landing = string }
load = string
orchestration = string variable "prefix" {
trasformation = string description = "Unique prefix used for resource names."
datalake-l0 = string type = string
datalake-l1 = string
datalake-l2 = string
datalake-playground = string
common = string
exposure = string
})
default = {
landing = "lnd"
load = "lod"
orchestration = "orc"
trasformation = "trf"
datalake-l0 = "dtl-0"
datalake-l1 = "dtl-1"
datalake-l2 = "dtl-2"
datalake-playground = "dtl-plg"
common = "cmn"
exposure = "exp"
}
} }
variable "project_services" { variable "project_services" {
@ -132,16 +97,10 @@ variable "project_services" {
] ]
} }
variable "location_config" { variable "region" {
description = "Locations where resources will be deployed. Map to configure region and multiregion specs." description = "Region used for regional resources."
type = object({ type = string
region = string default = "europe-west1"
multi_region = string
})
default = {
region = "europe-west1"
multi_region = "eu"
}
} }
variable "service_encryption_keys" { # service encription key variable "service_encryption_keys" { # service encription key