zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix variabler to support existing projects.

This commit is contained in:
lcaggio 2023-02-20 00:39:37 +01:00
parent ad6667a8fa
commit c8f25512eb
9 changed files with 136 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
View File

@ -15,37 +15,37 @@
# tfdoc:file:description drop off project and resources.
locals {
drop_orch_service_accounts = [
module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
]
}
module "drop-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
group_iam = {
group_iam_drp = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/pubsub.editor",
"roles/storage.admin",
]
}
iam = {
iam_drp = {
"roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = concat(
local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email]
)
"roles/pubsub.subscriber" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
}
}
module "drop-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
# group_iam = local.group_iam_drp
iam = var.project_config.billing_account_id != null ? local.iam_drp : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",

43
blueprints/data-solutions/data-platform-foundations/02-load.tf
View File

@ -15,6 +15,22 @@
# tfdoc:file:description Load project and VPC.
locals {
group_iam_load = {
(local.groups.data-engineers) = [
"roles/compute.viewer",
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam_load = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
load_service_accounts = [
"serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
module.load-sa-df-0.iam_email
@ -35,27 +51,14 @@ locals {
module "load-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/compute.viewer",
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
# group_iam = local.group_iam_load
iam = var.project_config.billing_account_id != null ? local.iam_load : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
@ -108,11 +111,11 @@ module "load-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-lod"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-lod"
region = var.region
}
]
@ -132,7 +135,7 @@ module "load-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-lod"
region = var.region
router_network = module.load-vpc.0.name
}

63
blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
View File

@ -15,31 +15,7 @@
# tfdoc:file:description Orchestration project and VPC.
locals {
orch_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)
# Note: This formatting is needed for output purposes since the fabric artifact registry
# module doesn't yet expose the docker usage path of a registry folder in the needed format.
orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}
module "orch-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
group_iam = {
group_iam_orch = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
@ -54,7 +30,7 @@ module "orch-project" {
"roles/serviceusage.serviceUsageConsumer",
]
}
iam = {
iam_orch = {
"roles/bigquery.dataEditor" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
@ -85,7 +61,34 @@ module "orch-project" {
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
oslogin = false
orch_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)
# Note: This formatting is needed for output purposes since the fabric artifact registry
# module doesn't yet expose the docker usage path of a registry folder in the needed format.
orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}
module "orch-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
# group_iam = local.group_iam_orch
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
oslogin = false
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
@ -133,11 +136,11 @@ module "orch-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.orch-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-orch"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-orch"
region = var.region
secondary_ip_ranges = {
pods = "10.10.8.0/22"
@ -161,7 +164,7 @@ module "orch-nat" {
count = local.use_shared_vpc ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.orch-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-orch"
region = var.region
router_network = module.orch-vpc.0.name
}

51
blueprints/data-solutions/data-platform-foundations/04-transformation.tf
View File

@ -15,32 +15,13 @@
# tfdoc:file:description Trasformation project and VPC.
locals {
transf_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.transf-vpc.0.subnet_self_links)[0]
)
transf_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.transf-vpc.0.self_link
)
}
module "transf-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
group_iam = {
group_iam_trf = {
(local.groups.data-engineers) = [
"roles/bigquery.jobUser",
"roles/dataflow.admin",
]
}
iam = {
iam_trf = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email,
]
@ -55,6 +36,28 @@ module "transf-project" {
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
]
}
transf_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.transf-vpc.0.subnet_self_links)[0]
)
transf_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.transf-vpc.0.self_link
)
}
module "transf-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
# group_iam = local.group_iam_trf
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
@ -132,11 +135,11 @@ module "transf-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-trf"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-trf"
region = var.region
}
]
@ -156,7 +159,7 @@ module "transf-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-trf"
region = var.region
router_network = module.transf-vpc.0.name
}

29
blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
View File

@ -82,14 +82,15 @@ locals {
module "dwh-lnd-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}"
group_iam = local.dwh_group_iam
iam = local.dwh_lnd_iam
services = local.dwh_services
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
@ -98,14 +99,15 @@ module "dwh-lnd-project" {
module "dwh-cur-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}"
group_iam = local.dwh_group_iam
iam = local.dwh_iam
services = local.dwh_services
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
@ -114,14 +116,15 @@ module "dwh-cur-project" {
module "dwh-conf-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}"
group_iam = local.dwh_group_iam
iam = local.dwh_iam
services = local.dwh_services
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_iam : null
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
@ -141,7 +144,7 @@ module "dwh-lnd-bq-0" {
module "dwh-cur-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dwh-cur-project.project_id
id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0"
id = "${replace(var.prefix, "-", "_")}_dwh_cur_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}

23
blueprints/data-solutions/data-platform-foundations/06-common.tf
View File

@ -14,14 +14,8 @@
# tfdoc:file:description common project.
module "common-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}"
group_iam = {
locals {
group_iam_common = {
(local.groups.data-analysts) = [
"roles/datacatalog.viewer",
]
@ -35,7 +29,7 @@ module "common-project" {
"roles/datacatalog.admin"
]
}
iam = {
iam_common = {
"roles/dlp.user" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email
@ -52,6 +46,17 @@ module "common-project" {
# local.groups_iam.data-analysts
]
}
}
module "common-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}"
# group_iam = local.group_iam_common
iam = var.project_config.billing_account_id != null ? local.iam_common : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null
services = concat(var.project_services, [
"datacatalog.googleapis.com",
"dlp.googleapis.com",

2
blueprints/data-solutions/data-platform-foundations/07-exposure.tf
View File

@ -16,7 +16,7 @@
module "exp-project" {
source = "../../../modules/project"
parent = var.folder_id
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix

4
blueprints/data-solutions/data-platform-foundations/README.md
View File

@ -233,6 +233,10 @@ To create Cloud Key Management keys in the Data Platform you can uncomment the C
To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level.
To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules.
### Project Configuration
The solution can be deployed creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly.
## Demo pipeline
The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features.

2
blueprints/data-solutions/data-platform-foundations/variables.tf
View File

@ -181,6 +181,7 @@ variable "project_config" {
description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
type = object({
billing_account_id = optional(string, null)
parent = string
project_ids = optional(object({
drop = string
load = string
@ -204,7 +205,6 @@ variable "project_config" {
}
)
})
default = {}
validation {
condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null
error_message = "At least one attribute should be set."