zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move first draft to fast branch

This commit is contained in:
Lorenzo Caggioni 2022-01-18 08:58:14 +01:00
parent 338422a6a8
commit e4d1db7def
19 changed files with 1346 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
examples/data-solutions/dp-foundation/01-lnd-datastorage.tf Normal file
View File

@ -0,0 +1,78 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
locals {
lnd_bucket_retention_policy = {
retention_period = 7776000 # 90 * 24 * 60 * 60
is_locked = false
}
}
###############################################################################
# GCS #
###############################################################################
module "lnd-sa-cs-0" {
source = "../../../modules/iam-service-account"
project_id = module.lnd-prj.project_id
name = "cs-0"
prefix = local.prefix_lnd
}
module "lnd-cs-0" {
source = "../../../modules/gcs"
project_id = module.lnd-prj.project_id
name = "cs-0"
prefix = local.prefix_lnd
location = var.region
storage_class = "REGIONAL"
retention_policy = local.lnd_bucket_retention_policy
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
force_destroy = var.data_force_destroy
}
###############################################################################
# PubSub #
###############################################################################
module "lnd-sa-ps-0" {
source = "../../../modules/iam-service-account"
project_id = module.lnd-prj.project_id
name = "ps-0"
prefix = local.prefix_lnd
}
module "lnd-ps-0" {
source = "../../../modules/pubsub"
project_id = module.lnd-prj.project_id
name = "${local.prefix_lnd}-ps-0"
}
###############################################################################
# BigQuery #
###############################################################################
module "lnd-sa-bq-0" {
source = "../../../modules/iam-service-account"
project_id = module.lnd-prj.project_id
name = "bq-0"
prefix = local.prefix_lnd
}
module "lnd-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.lnd-prj.project_id
id = "${replace(local.prefix_lnd, "-", "_")}_bq_0"
location = var.region
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
}

92
examples/data-solutions/dp-foundation/01-lnd-main.tf.tf Normal file
View File

@ -0,0 +1,92 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
locals {
group_iam_lnd = {
"${local.groups.data-engineers}" = [
"roles/bigquery.dataEditor",
"roles/pubsub.editor",
"roles/storage.admin",
"roles/storage.objectViewer",
"roles/viewer",
],
# "${local.groups.data-scientists}" = [
# "roles/bigquery.dataViewer",
# "roles/bigquery.jobUser",
# "roles/bigquery.user",
# "roles/pubsub.viewer",
# ]
}
iam_lnd = {
"roles/bigquery.dataEditor" = [
module.lnd-sa-bq-0.iam_email,
]
"roles/bigquery.dataViewer" = [
module.lod-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email,
]
"roles/bigquery.jobUser" = [
module.orc-sa-cmp-0.iam_email
]
"roles/bigquery.user" = [
module.lod-sa-df-0.iam_email
]
"roles/pubsub.publisher" = [
module.lnd-sa-ps-0.iam_email
]
"roles/pubsub.subscriber" = [
module.lod-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email
]
"roles/storage.objectAdmin" = [
module.lod-sa-df-0.iam_email,
]
"roles/storage.objectCreator" = [
module.lnd-sa-cs-0.iam_email,
]
"roles/storage.objectViewer" = [
module.orc-sa-cmp-0.iam_email,
]
"roles/storage.admin" = [
module.lod-sa-df-0.iam_email,
]
}
prefix_lnd = "${var.prefix}-lnd"
}
###############################################################################
# Project #
###############################################################################
module "lnd-prj" {
source = "../../../modules/project"
name = var.project_id["landing"]
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_lnd : {}
iam_additive = var.project_create == null ? local.iam_lnd : {}
# group_iam = local.group_iam_lnd
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"pubsub.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com",
])
}

42
examples/data-solutions/dp-foundation/02-lod-datastorage.tf Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# GCS #
###############################################################################
module "lod-sa-df-0" {
source = "../../../modules/iam-service-account"
project_id = module.lod-prj.project_id
name = "lod-df-0"
prefix = local.prefix_lod
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
],
"roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email,
]
}
}
module "lod-cs-df-0" {
source = "../../../modules/gcs"
project_id = module.lod-prj.project_id
name = "lod-cs-0"
prefix = local.prefix_lod
storage_class = "REGIONAL"
location = var.region
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
}

75
examples/data-solutions/dp-foundation/02-lod-main.tf Normal file
View File

@ -0,0 +1,75 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
locals {
group_iam_lod = {
"${local.groups.data-engineers}" = [
"roles/compute.viewer",
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam_lod = {
"roles/bigquery.jobUser" = [
module.lod-sa-df-0.iam_email
]
"roles/compute.serviceAgent" = [
"serviceAccount:${module.lod-prj.service_accounts.robots.compute}"
]
"roles/dataflow.admin" = [
module.orc-sa-cmp-0.iam_email,
module.lod-sa-df-0.iam_email
]
"roles/dataflow.worker" = [
module.lod-sa-df-0.iam_email
]
"roles/dataflow.serviceAgent" = [
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}"
]
"roles/storage.objectAdmin" = [
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}"
]
}
prefix_lod = "${var.prefix}-lod"
}
###############################################################################
# Project #
###############################################################################
module "lod-prj" {
source = "../../../modules/project"
name = var.project_id["load"]
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_lod : {}
iam_additive = var.project_create == null ? local.iam_lod : {}
# group_iam = local.group_iam_lod
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
}

49
examples/data-solutions/dp-foundation/02-lod-vpc.tf Normal file
View File

@ -0,0 +1,49 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# Network #
###############################################################################
module "lod-vpc" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-vpc"
project_id = module.lod-prj.project_id
name = "${local.prefix_lod}-lod-vpc"
subnets = [
{
ip_cidr_range = var.network_config.vpc_subnet_range.load
name = "subnet"
region = var.region
secondary_ip_range = {}
}
]
}
module "lod-vpc-firewall" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-vpc-firewall"
project_id = module.lod-prj.project_id
network = module.lod-vpc[0].name
admin_ranges = values(module.lod-vpc[0].subnet_ips)
}
module "lod-nat" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.lod-prj.project_id
region = var.region
name = "${local.prefix_lod}-default"
router_network = module.lod-vpc[0].name
}

129
examples/data-solutions/dp-foundation/03-orc-composer.tf Normal file
View File

@ -0,0 +1,129 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# Composer #
###############################################################################
module "orc-sa-cmp-0" {
source = "../../../modules/iam-service-account"
project_id = module.orc-prj.project_id
name = "cmp-0"
prefix = local.prefix_orc
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
],
"roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email,
]
}
}
resource "google_composer_environment" "orc-cmp-0" {
name = "${local.prefix_orc}-cmp-0"
region = var.composer_config.region
provider = google-beta
project = module.orc-prj.project_id
config {
node_count = 3
node_config {
zone = "${var.composer_config.region}-b"
service_account = module.orc-sa-cmp-0.email
network = module.orc-vpc[0].self_link
subnetwork = module.orc-vpc[0].subnet_self_links["${var.composer_config.region}/subnet"]
tags = ["composer-worker", "http-server", "https-server"]
ip_allocation_policy {
use_ip_aliases = "true"
cluster_secondary_range_name = "pods"
services_secondary_range_name = "services"
}
}
software_config {
env_variables = {
DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id
DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id
DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id
DTL_EXP_BQ_DATASET = module.dtl-exp-bq-0.dataset_id
DTL_L0_GCS = module.dtl-0-cs-0.url
DTL_L1_GCS = module.dtl-1-cs-0.url
DTL_L2_GCS = module.dtl-2-cs-0.url
DTL_EXP_GCS = module.dtl-exp-cs-0.url
LND_GCS = module.lnd-cs-0.url
LND_BQ = module.lnd-bq-0.dataset_id
LND_PS = module.lnd-ps-0.id
LOD_GCS_STAGING = module.lod-cs-df-0.url
TRF_GCS_STAGING = module.trf-cs-df-0.url
GCP_REGION = var.composer_config.region
NET_VPC = module.orc-vpc[0].self_link
NET_SUBNET = module.orc-vpc[0].subnet_self_links["${var.composer_config.region}/subnet"]
DTL_L0_PRJ = module.dtl-0-prj.project_id
DTL_L1_PRJ = module.dtl-1-prj.project_id
DTL_L2_PRJ = module.dtl-2-prj.project_id
LND_PRJ = module.lnd-prj.project_id
LOD_PRJ = module.lod-prj.project_id
ORC_PRJ = module.orc-prj.project_id
TRF_PRJ = module.trf-prj.project_id
LOD_SA_DF = module.lod-sa-df-0.email
TRF_SA_DF = module.lod-sa-df-0.email
}
}
private_environment_config {
enable_private_endpoint = "true"
master_ipv4_cidr_block = var.composer_config.ip_range_gke_master
cloud_sql_ipv4_cidr_block = var.composer_config.ip_range_cloudsql
web_server_ipv4_cidr_block = var.composer_config.ip_range_web_server
}
dynamic "encryption_config" {
for_each = can(module.kms[0].keys.key-cmp.id) ? { 1 = 1 } : {}
content {
kms_key_name = var.cmek_encryption ? try(module.kms[0].keys.key-cmp.id, null) : null
}
}
# web_server_network_access_control {
# allowed_ip_range {
# value = "172.16.0.0/12"
# description = "Allowed ip range"
# }
# }
}
depends_on = [
module.dtl-0-bq-0,
module.dtl-1-bq-0,
module.dtl-2-bq-0,
module.dtl-exp-bq-0,
module.dtl-0-cs-0,
module.dtl-1-cs-0,
module.dtl-2-cs-0,
module.dtl-exp-cs-0,
module.lnd-cs-0,
module.lnd-bq-0,
module.lnd-ps-0,
module.lod-cs-df-0,
module.trf-cs-df-0,
module.orc-vpc,
module.orc-vpc,
module.dtl-0-prj,
module.dtl-1-prj,
module.dtl-2-prj,
module.lnd-prj,
module.lod-prj,
module.orc-prj,
module.trf-prj,
module.lod-sa-df-0,
module.lod-sa-df-0,
]
}

27
examples/data-solutions/dp-foundation/03-orc-datastorage.tf Normal file
View File

@ -0,0 +1,27 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# GCS #
###############################################################################
module "orc-cs-0" {
source = "../../../modules/gcs"
project_id = module.orc-prj.project_id
name = "orc-cs-0"
prefix = local.prefix_orc
location = var.region
storage_class = "REGIONAL"
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
}

90
examples/data-solutions/dp-foundation/03-orc-main.tf Normal file
View File

@ -0,0 +1,90 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
locals {
iam_orc = {
"roles/bigquery.dataEditor" = [
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email,
]
"roles/bigquery.jobUser" = [
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email,
]
"roles/composer.worker" = [
module.orc-sa-cmp-0.iam_email
]
"roles/compute.networkUser" = [
module.orc-sa-cmp-0.iam_email,
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}",
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}",
"serviceAccount:${module.orc-prj.service_accounts.cloud_services}"
]
"roles/storage.objectAdmin" = [
module.lod-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email,
]
"roles/storage.admin" = [
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email
]
}
group_iam_orc = {
"${local.groups.data-engineers}" = [
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
"roles/cloudbuild.builds.editor",
"roles/composer.admin",
"roles/composer.environmentAndStorageObjectAdmin",
"roles/iap.httpsResourceAccessor",
"roles/compute.networkUser",
"roles/storage.objectAdmin",
"roles/storage.admin",
"roles/compute.networkUser"
]
}
prefix_orc = "${var.prefix}-orc"
}
module "orc-prj" {
source = "../../../modules/project"
name = var.project_id["orchestration"]
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_orc : {}
iam_additive = var.project_create == null ? local.iam_orc : {}
group_iam = local.group_iam_orc
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"composer.googleapis.com",
"container.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
}

53
examples/data-solutions/dp-foundation/03-orc-vpc.tf Normal file
View File

@ -0,0 +1,53 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# Network #
###############################################################################
module "orc-vpc" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-vpc"
project_id = module.orc-prj.project_id
name = "${local.prefix_orc}-orc-vpc"
subnets = [
{
ip_cidr_range = var.network_config.vpc_subnet_range.orchestration
name = "subnet"
region = var.region
secondary_ip_range = {}
secondary_ip_range = {
pods = var.composer_config.secondary_ip_range.pods
services = var.composer_config.secondary_ip_range.services
}
}
]
}
module "orc-vpc-firewall" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-vpc-firewall"
project_id = module.orc-prj.project_id
network = module.orc-vpc[0].name
admin_ranges = values(module.orc-vpc[0].subnet_ips)
}
module "orc-nat" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.orc-prj.project_id
region = var.region
name = "${local.prefix_orc}-default"
router_network = module.orc-vpc[0].name
}

61
examples/data-solutions/dp-foundation/04-trf-datastorage.tf Normal file
View File

@ -0,0 +1,61 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# GCS #
###############################################################################
module "trf-sa-df-0" {
source = "../../../modules/iam-service-account"
project_id = module.trf-prj.project_id
name = "trf-df-0"
prefix = local.prefix_trf
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
],
"roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email,
]
}
}
module "trf-cs-df-0" {
source = "../../../modules/gcs"
project_id = module.trf-prj.project_id
name = "trf-cs-0"
prefix = local.prefix_trf
location = var.region
storage_class = "REGIONAL"
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
}
###############################################################################
# BQ #
###############################################################################
module "trf-sa-bq-0" {
source = "../../../modules/iam-service-account"
project_id = module.trf-prj.project_id
name = "trf-bq-0"
prefix = local.prefix_trf
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
],
"roles/iam.serviceAccountUser" = [
module.orc-sa-cmp-0.iam_email,
]
}
}

72
examples/data-solutions/dp-foundation/04-trf-main.tf Normal file
View File

@ -0,0 +1,72 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
locals {
group_iam_trf = {
"${local.groups.data-engineers}" = [
"roles/bigquery.jobUser",
"roles/dataflow.admin",
"roles/viewer",
]
}
iam_trf = {
"roles/bigquery.dataViewer" = [
module.orc-sa-cmp-0.iam_email
]
"roles/bigquery.jobUser" = [
module.trf-sa-bq-0.iam_email,
]
"roles/dataflow.admin" = [
module.orc-sa-cmp-0.iam_email,
]
"roles/dataflow.worker" = [
module.trf-sa-df-0.iam_email
]
"roles/storage.objectAdmin" = [
module.trf-sa-df-0.iam_email,
module.orc-sa-cmp-0.iam_email,
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}"
]
}
prefix_trf = "${var.prefix}-trf"
}
###############################################################################
# Project #
###############################################################################
module "trf-prj" {
source = "../../../modules/project"
name = var.project_id["trasformation"]
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_trf : {}
iam_additive = var.project_create == null ? local.iam_trf : {}
group_iam = local.group_iam_trf
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
}

49
examples/data-solutions/dp-foundation/04-trf-vpc.tf Normal file
View File

@ -0,0 +1,49 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# Network #
###############################################################################
module "trf-vpc" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-vpc"
project_id = module.trf-prj.project_id
name = "${local.prefix_trf}-trf-vpc"
subnets = [
{
ip_cidr_range = var.network_config.vpc_subnet_range.transformation
name = "subnet"
region = var.region
secondary_ip_range = {}
}
]
}
module "trf-vpc-firewall" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-vpc-firewall"
project_id = module.trf-prj.project_id
network = module.trf-vpc[0].name
admin_ranges = values(module.orc-vpc[0].subnet_ips)
}
module "trf-nat" {
count = var.network_config.network != null ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.trf-prj.project_id
region = var.region
name = "${local.prefix_trf}-default"
router_network = module.trf-vpc[0].name
}

97
examples/data-solutions/dp-foundation/05-dtl-datastorage.tf Normal file
View File

@ -0,0 +1,97 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# BQ #
###############################################################################
module "dtl-0-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dtl-0-prj.project_id
id = "${replace(local.prefix_lnd, "-", "_")}_0_bq_0"
location = var.region
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
}
module "dtl-1-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dtl-1-prj.project_id
id = "${replace(local.prefix_lnd, "-", "_")}_1_bq_0"
location = var.region
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
}
module "dtl-2-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dtl-2-prj.project_id
id = "${replace(local.prefix_lnd, "-", "_")}_2_bq_0"
location = var.region
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
}
module "dtl-exp-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dtl-exp-prj.project_id
id = "${replace(local.prefix_lnd, "-", "_")}_exp_bq_0"
location = var.region
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
}
###############################################################################
# GCS #
###############################################################################
module "dtl-0-cs-0" {
source = "../../../modules/gcs"
project_id = module.dtl-0-prj.project_id
name = "0-cs-0"
prefix = local.prefix_dtl
location = var.region
storage_class = "REGIONAL"
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
force_destroy = var.data_force_destroy
}
module "dtl-1-cs-0" {
source = "../../../modules/gcs"
project_id = module.dtl-1-prj.project_id
name = "1-cs-0"
prefix = local.prefix_dtl
location = var.region
storage_class = "REGIONAL"
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
force_destroy = var.data_force_destroy
}
module "dtl-2-cs-0" {
source = "../../../modules/gcs"
project_id = module.dtl-2-prj.project_id
name = "2-cs-0"
prefix = local.prefix_dtl
location = var.region
storage_class = "REGIONAL"
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
force_destroy = var.data_force_destroy
}
module "dtl-exp-cs-0" {
source = "../../../modules/gcs"
project_id = module.dtl-exp-prj.project_id
name = "exp-cs-0"
prefix = local.prefix_dtl
location = var.region
storage_class = "REGIONAL"
encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
force_destroy = var.data_force_destroy
}

160
examples/data-solutions/dp-foundation/05-dtl-main.tf Normal file
View File

@ -0,0 +1,160 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
locals {
group_iam_dtl = {
"${local.groups.data-engineers}" = [
"roles/bigquery.dataEditor",
"roles/storage.admin",
"roles/storage.objectCreator",
"roles/storage.objectViewer",
"roles/viewer",
],
# "${local.groups.data-scientists}" = [
# "roles/bigquery.jobUser",
# ]
}
iam_dtl = {
"roles/bigquery.dataEditor" = [
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
module.trf-sa-bq-0.iam_email,
module.orc-sa-cmp-0.iam_email,
]
"roles/bigquery.jobUser" = [
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
]
"roles/storage.admin" = [
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
]
"roles/storage.objectCreator" = [
module.lod-sa-df-0.iam_email,
module.trf-sa-df-0.iam_email,
module.trf-sa-bq-0.iam_email,
module.orc-sa-cmp-0.iam_email,
]
"roles/storage.objectViewer" = [
module.trf-sa-df-0.iam_email,
module.trf-sa-bq-0.iam_email,
module.orc-sa-cmp-0.iam_email,
]
}
prefix_dtl = "${var.prefix}-dtl"
}
###############################################################################
# Project #
###############################################################################
module "dtl-0-prj" {
source = "../../../modules/project"
name = "${var.project_id["datalake"]}-0"
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_dtl : {}
iam_additive = var.project_create == null ? local.iam_dtl : {}
group_iam = local.group_iam_dtl
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
}
module "dtl-1-prj" {
source = "../../../modules/project"
name = "${var.project_id["datalake"]}-1"
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_dtl : {}
iam_additive = var.project_create == null ? local.iam_dtl : {}
group_iam = local.group_iam_dtl
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
}
module "dtl-2-prj" {
source = "../../../modules/project"
name = "${var.project_id["datalake"]}-2"
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_dtl : {}
iam_additive = var.project_create == null ? local.iam_dtl : {}
group_iam = local.group_iam_dtl
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
}
module "dtl-exp-prj" {
source = "../../../modules/project"
name = "${var.project_id["datalake"]}-exp"
parent = try(var.project_create.parent, null)
billing_account = try(var.project_create.billing_account_id, null)
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_dtl : {}
iam_additive = var.project_create == null ? local.iam_dtl : {}
group_iam = local.group_iam_dtl
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com"
])
}

77
examples/data-solutions/dp-foundation/99-kms-main.tf Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
module "kms" {
count = var.cmek_encryption ? 1 : 0
source = "../../../modules/kms"
project_id = module.lnd-prj.project_id
keyring = {
name = "${var.prefix}-keyring",
location = var.region
}
keys = {
key-bq = null
key-cmp = null
key-df = null
key-gcs = null
key-ps = null
}
key_iam = {
key-bq = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.lnd-prj.service_accounts.robots.bq}",
"serviceAccount:${module.dtl-0-prj.service_accounts.robots.bq}",
"serviceAccount:${module.dtl-1-prj.service_accounts.robots.bq}",
"serviceAccount:${module.dtl-2-prj.service_accounts.robots.bq}",
"serviceAccount:${module.dtl-exp-prj.service_accounts.robots.bq}",
]
},
key-cmp = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.orc-prj.service_accounts.robots.artifactregistry}",
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}",
"serviceAccount:${module.orc-prj.service_accounts.robots.compute}",
"serviceAccount:${module.orc-prj.service_accounts.robots.composer}",
"serviceAccount:${module.orc-prj.service_accounts.robots.pubsub}",
"serviceAccount:${module.orc-prj.service_accounts.robots.storage}",
]
},
key-df = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
"serviceAccount:${module.lod-prj.service_accounts.robots.compute}",
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}",
"serviceAccount:${module.trf-prj.service_accounts.robots.compute}",
]
}
key-gcs = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.dtl-0-prj.service_accounts.robots.storage}",
"serviceAccount:${module.dtl-1-prj.service_accounts.robots.storage}",
"serviceAccount:${module.dtl-2-prj.service_accounts.robots.storage}",
"serviceAccount:${module.dtl-exp-prj.service_accounts.robots.storage}",
"serviceAccount:${module.lnd-prj.service_accounts.robots.storage}",
"serviceAccount:${module.lod-prj.service_accounts.robots.storage}",
"serviceAccount:${module.orc-prj.service_accounts.robots.storage}",
"serviceAccount:${module.trf-prj.service_accounts.robots.storage}",
]
},
key-ps = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.lnd-prj.service_accounts.robots.pubsub}"
]
}
}
}

15
examples/data-solutions/dp-foundation/README.md Normal file
View File

@ -0,0 +1,15 @@
- GCS and BQ regional
- KMS: Regional keyring, one key per product
- Composer require "Require OS Login" not enforced
- Groups: gcp-data-scientists, gcp-data-engineers
#TODO KMS: support key per product
#TODO Write README
#TODO Run a working test
#TODO Write a working e2e test
#TODO Column level access on BQ
#TODO DataCatalog
#TODO DLP
#TODO DataLake layers: Tables, views and Authorized views
#TODO ShareVPC Role: roles/composer.sharedVpcAgent, roles/container.hostServiceAgentUser
#TODO Composer require "Require OS Login" not enforced

30
examples/data-solutions/dp-foundation/backend.tf.sample Normal file
View File

@ -0,0 +1,30 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The `impersonate_service_account` option require the identity launching terraform
# role `roles/iam.serviceAccountTokenCreator` on the Service Account specified.
terraform {
backend "gcs" {
bucket = "BUCKET_NAME"
prefix = "PREFIX"
impersonate_service_account = "SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com"
}
}
provider "google" {
impersonate_service_account = "SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com"
}
provider "google-beta" {
impersonate_service_account = "SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com"
}

18
examples/data-solutions/dp-foundation/main.tf Normal file
View File

@ -0,0 +1,18 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
locals {
groups = { for k, v in var.groups : k => "${v}@${var.organization.domain}" }
groups_iam = { for k, v in local.groups : k => "group:${v}" }
}

132
examples/data-solutions/dp-foundation/variables.tf Normal file
View File

@ -0,0 +1,132 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
variable "cmek_encryption" {
description = "Flag to enable CMEK on GCP resources created."
type = bool
default = false
}
variable "composer_config" {
type = object({
ip_range_cloudsql = string
ip_range_gke_master = string
ip_range_web_server = string
region = string
secondary_ip_range = object({
pods = string
services = string
})
})
default = {
ip_range_cloudsql = "10.20.10.0/24"
ip_range_gke_master = "10.20.11.0/28"
ip_range_web_server = "10.20.11.16/28"
region = "europe-west1"
secondary_ip_range = {
pods = "10.10.8.0/22"
services = "10.10.12.0/24"
}
}
}
variable "data_force_destroy" {
description = "Flag to set 'force_destroy' on data services like biguqery or cloud storage."
type = bool
default = false
}
variable "groups" {
description = "Groups."
type = map(string)
default = {
data-engineers = "gcp-data-engineers"
data-scientists = "gcp-data-scientists"
}
}
variable "network_config" {
description = "Shared VPC to use. If not null networks will be created in projects."
type = object({
network = string
vpc_subnet_range = object({
load = string
transformation = string
orchestration = string
})
})
default = {
network = null
vpc_subnet_range = {
load = "10.10.0.0/24"
transformation = "10.10.0.0/24"
orchestration = "10.10.0.0/24"
}
}
}
variable "organization" {
description = "Organization details."
type = object({
domain = string
})
}
variable "prefix" {
description = "Unique prefix used for resource names. Not used for project if 'project_create' is null."
type = string
}
variable "project_create" {
description = "Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format"
type = object({
billing_account_id = string
parent = string
})
default = null
}
variable "project_id" {
description = "Project id, references existing project if `project_create` is null."
type = object({
landing = string
load = string
orchestration = string
trasformation = string
datalake = string
})
default = {
landing = "lnd"
load = "lod"
orchestration = "orc"
trasformation = "trf"
datalake = "dtl"
}
}
variable "project_services" {
type = list(string)
default = [
"cloudresourcemanager.googleapis.com",
"iam.googleapis.com",
"serviceusage.googleapis.com",
"stackdriver.googleapis.com"
]
}
variable "region" {
description = "The region where resources will be deployed."
type = string
default = "europe-west1"
}