zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1165 from GoogleCloudPlatform/lcaggio/dp-projectcreate 

DataPlatform: Support project creation
This commit is contained in:
lcaggio 2023-02-23 12:10:43 +01:00 committed by GitHub
parent ad0840656b 47855cb682
commit 89da766801
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 284 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
View File

@ -15,36 +15,34 @@
# tfdoc:file:description drop off project and resources. # tfdoc:file:description drop off project and resources.
locals { locals {
drop_orch_service_accounts = [ iam_drp = {
module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email "roles/bigquery.dataEditor" = [
] module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers
]
"roles/bigquery.user" = [
module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
"roles/storage.objectAdmin" = [
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email
]
}
} }
module "drop-project" { module "drop-project" {
source = "../../../modules/project" source = "../../../modules/project"
parent = var.folder_id parent = var.project_config.parent
billing_account = var.billing_account_id billing_account = var.project_config.billing_account_id
prefix = var.prefix project_create = var.project_config.billing_account_id != null
name = "drp${local.project_suffix}" prefix = var.project_config.billing_account_id == null ? null : var.prefix
group_iam = { name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
(local.groups.data-engineers) = [ iam = var.project_config.billing_account_id != null ? local.iam_drp : null
"roles/bigquery.dataEditor", iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
"roles/pubsub.editor",
"roles/storage.admin",
]
}
iam = {
"roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = concat(
local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email]
)
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
}
services = concat(var.project_services, [ services = concat(var.project_services, [
"bigquery.googleapis.com", "bigquery.googleapis.com",
"bigqueryreservation.googleapis.com", "bigqueryreservation.googleapis.com",

55
blueprints/data-solutions/data-platform-foundations/02-load.tf
View File

@ -15,6 +15,19 @@
# tfdoc:file:description Load project and VPC. # tfdoc:file:description Load project and VPC.
locals { locals {
iam_load = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email,
module.load-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/dataflow.developer" = [
local.groups_iam.data-engineers
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
load_service_accounts = [ load_service_accounts = [
"serviceAccount:${module.load-project.service_accounts.robots.dataflow}", "serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
module.load-sa-df-0.iam_email module.load-sa-df-0.iam_email
@ -35,26 +48,13 @@ locals {
module "load-project" { module "load-project" {
source = "../../../modules/project" source = "../../../modules/project"
parent = var.folder_id parent = var.project_config.parent
billing_account = var.billing_account_id billing_account = var.project_config.billing_account_id
prefix = var.prefix project_create = var.project_config.billing_account_id != null
name = "lod${local.project_suffix}" prefix = var.project_config.billing_account_id == null ? null : var.prefix
group_iam = { name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}"
(local.groups.data-engineers) = [ iam = var.project_config.billing_account_id != null ? local.iam_load : null
"roles/compute.viewer", iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
services = concat(var.project_services, [ services = concat(var.project_services, [
"bigquery.googleapis.com", "bigquery.googleapis.com",
"bigqueryreservation.googleapis.com", "bigqueryreservation.googleapis.com",
@ -86,8 +86,13 @@ module "load-sa-df-0" {
name = "load-df-0" name = "load-df-0"
display_name = "Data platform Dataflow load service account" display_name = "Data platform Dataflow load service account"
iam = { iam = {
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] "roles/iam.serviceAccountTokenCreator" = [
"roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email] local.groups_iam.data-engineers,
module.orch-sa-cmp-0.iam_email
],
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
]
} }
} }
@ -107,11 +112,11 @@ module "load-vpc" {
source = "../../../modules/net-vpc" source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1 count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id project_id = module.load-project.project_id
name = "${var.prefix}-default" name = "${var.prefix}-lod"
subnets = [ subnets = [
{ {
ip_cidr_range = "10.10.0.0/24" ip_cidr_range = "10.10.0.0/24"
name = "default" name = "${var.prefix}-lod"
region = var.region region = var.region
} }
] ]
@ -131,7 +136,7 @@ module "load-nat" {
source = "../../../modules/net-cloudnat" source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1 count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id project_id = module.load-project.project_id
name = "${var.prefix}-default" name = "${var.prefix}-lod"
region = var.region region = var.region
router_network = module.load-vpc.0.name router_network = module.load-vpc.0.name
} }

101
blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
View File

@ -15,6 +15,42 @@
# tfdoc:file:description Orchestration project and VPC. # tfdoc:file:description Orchestration project and VPC.
locals { locals {
iam_orch = {
"roles/artifactregistry.admin" = [local.groups_iam.data-engineers]
"roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email]
"roles/bigquery.dataEditor" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [
module.orch-sa-cmp-0.iam_email,
local.groups_iam.data-engineers
]
"roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers]
"roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email]
"roles/composer.admin" = [local.groups_iam.data-engineers]
"roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers]
"roles/composer.ServiceAgentV2Ext" = [
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
]
"roles/composer.worker" = [
module.orch-sa-cmp-0.iam_email
]
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
"roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers]
"roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers]
"roles/storage.objectAdmin" = [
module.orch-sa-cmp-0.iam_email,
module.orch-sa-df-build.iam_email,
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
"serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
local.groups_iam.data-engineers
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
orch_subnet = ( orch_subnet = (
local.use_shared_vpc local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration ? var.network_config.subnet_self_links.orchestration
@ -34,57 +70,14 @@ locals {
module "orch-project" { module "orch-project" {
source = "../../../modules/project" source = "../../../modules/project"
parent = var.folder_id parent = var.project_config.parent
billing_account = var.billing_account_id billing_account = var.project_config.billing_account_id
prefix = var.prefix project_create = var.project_config.billing_account_id != null
name = "orc${local.project_suffix}" prefix = var.project_config.billing_account_id == null ? null : var.prefix
group_iam = { name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
(local.groups.data-engineers) = [ iam = var.project_config.billing_account_id != null ? local.iam_orch : null
"roles/bigquery.dataEditor", iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
"roles/bigquery.jobUser", oslogin = false
"roles/cloudbuild.builds.editor",
"roles/composer.admin",
"roles/composer.environmentAndStorageObjectAdmin",
"roles/iap.httpsResourceAccessor",
"roles/iam.serviceAccountUser",
"roles/storage.objectAdmin",
"roles/storage.admin",
"roles/artifactregistry.admin",
"roles/serviceusage.serviceUsageConsumer",
]
}
iam = {
"roles/bigquery.dataEditor" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
]
"roles/bigquery.jobUser" = [
module.orch-sa-cmp-0.iam_email,
]
"roles/composer.ServiceAgentV2Ext" = [
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
]
"roles/composer.worker" = [
module.orch-sa-cmp-0.iam_email
]
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
]
"roles/storage.objectAdmin" = [
module.orch-sa-cmp-0.iam_email,
module.orch-sa-df-build.iam_email,
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
"serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
]
"roles/artifactregistry.reader" = [
module.load-sa-df-0.iam_email,
]
"roles/cloudbuild.serviceAgent" = [
module.orch-sa-df-build.iam_email,
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
oslogin = false
services = concat(var.project_services, [ services = concat(var.project_services, [
"artifactregistry.googleapis.com", "artifactregistry.googleapis.com",
"bigquery.googleapis.com", "bigquery.googleapis.com",
@ -132,11 +125,11 @@ module "orch-vpc" {
source = "../../../modules/net-vpc" source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1 count = local.use_shared_vpc ? 0 : 1
project_id = module.orch-project.project_id project_id = module.orch-project.project_id
name = "${var.prefix}-default" name = "${var.prefix}-orch"
subnets = [ subnets = [
{ {
ip_cidr_range = "10.10.0.0/24" ip_cidr_range = "10.10.0.0/24"
name = "default" name = "${var.prefix}-orch"
region = var.region region = var.region
secondary_ip_ranges = { secondary_ip_ranges = {
pods = "10.10.8.0/22" pods = "10.10.8.0/22"
@ -160,7 +153,7 @@ module "orch-nat" {
count = local.use_shared_vpc ? 0 : 1 count = local.use_shared_vpc ? 0 : 1
source = "../../../modules/net-cloudnat" source = "../../../modules/net-cloudnat"
project_id = module.orch-project.project_id project_id = module.orch-project.project_id
name = "${var.prefix}-default" name = "${var.prefix}-orch"
region = var.region region = var.region
router_network = module.orch-vpc.0.name router_network = module.orch-vpc.0.name
} }

51
blueprints/data-solutions/data-platform-foundations/04-transformation.tf
View File

@ -15,6 +15,19 @@
# tfdoc:file:description Trasformation project and VPC. # tfdoc:file:description Trasformation project and VPC.
locals { locals {
iam_trf = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers
]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
"roles/dataflow.worker" = [module.transf-sa-df-0.iam_email]
"roles/storage.objectAdmin" = [
module.transf-sa-df-0.iam_email,
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
]
}
transf_subnet = ( transf_subnet = (
local.use_shared_vpc local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration ? var.network_config.subnet_self_links.orchestration
@ -29,31 +42,13 @@ locals {
module "transf-project" { module "transf-project" {
source = "../../../modules/project" source = "../../../modules/project"
parent = var.folder_id parent = var.project_config.parent
billing_account = var.billing_account_id billing_account = var.project_config.billing_account_id
prefix = var.prefix project_create = var.project_config.billing_account_id != null
name = "trf${local.project_suffix}" prefix = var.project_config.billing_account_id == null ? null : var.prefix
group_iam = { name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
(local.groups.data-engineers) = [ iam = var.project_config.billing_account_id != null ? local.iam_trf : null
"roles/bigquery.jobUser", iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null
"roles/dataflow.admin",
]
}
iam = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email,
]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email,
]
"roles/dataflow.worker" = [
module.transf-sa-df-0.iam_email
]
"roles/storage.objectAdmin" = [
module.transf-sa-df-0.iam_email,
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
]
}
services = concat(var.project_services, [ services = concat(var.project_services, [
"bigquery.googleapis.com", "bigquery.googleapis.com",
"bigqueryreservation.googleapis.com", "bigqueryreservation.googleapis.com",
@ -131,11 +126,11 @@ module "transf-vpc" {
source = "../../../modules/net-vpc" source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1 count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id project_id = module.transf-project.project_id
name = "${var.prefix}-default" name = "${var.prefix}-trf"
subnets = [ subnets = [
{ {
ip_cidr_range = "10.10.0.0/24" ip_cidr_range = "10.10.0.0/24"
name = "default" name = "${var.prefix}-trf"
region = var.region region = var.region
} }
] ]
@ -155,7 +150,7 @@ module "transf-nat" {
source = "../../../modules/net-cloudnat" source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1 count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id project_id = module.transf-project.project_id
name = "${var.prefix}-default" name = "${var.prefix}-trf"
region = var.region region = var.region
router_network = module.transf-vpc.0.name router_network = module.transf-vpc.0.name
} }

89
blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
View File

@ -15,54 +15,48 @@
# tfdoc:file:description Data Warehouse projects. # tfdoc:file:description Data Warehouse projects.
locals { locals {
dwh_group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/storage.admin",
],
(local.groups.data-analysts) = [
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser",
"roles/bigquery.metadataViewer",
"roles/bigquery.user",
"roles/datacatalog.viewer",
"roles/datacatalog.tagTemplateViewer",
"roles/storage.objectViewer",
]
}
dwh_lnd_iam = { dwh_lnd_iam = {
"roles/bigquery.dataOwner" = [ "roles/bigquery.dataOwner" = [
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
]
"roles/bigquery.dataViewer" = [
module.transf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email, module.transf-sa-bq-0.iam_email,
local.groups_iam.data-engineers
] ]
"roles/bigquery.jobUser" = [ "roles/bigquery.jobUser" = [
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
]
"roles/datacatalog.categoryAdmin" = [
module.transf-sa-bq-0.iam_email
]
"roles/storage.objectCreator" = [
module.load-sa-df-0.iam_email,
] ]
"roles/datacatalog.categoryAdmin" = [module.transf-sa-bq-0.iam_email]
"roles/datacatalog.tagTemplateViewer" = [local.groups_iam.data-engineers]
"roles/datacatalog.viewer" = [local.groups_iam.data-engineers]
"roles/storage.objectCreator" = [module.load-sa-df-0.iam_email]
"roles/storage.objectViewer" = [local.groups_iam.data-engineers]
} }
dwh_iam = { dwh_iam = {
"roles/bigquery.dataOwner" = [ "roles/bigquery.dataOwner" = [
module.transf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email, module.transf-sa-bq-0.iam_email,
] ]
"roles/bigquery.dataViewer" = [
local.groups_iam.data-analysts,
local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [ "roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email, module.transf-sa-bq-0.iam_email,
local.groups_iam.data-analysts,
local.groups_iam.data-engineers
] ]
"roles/datacatalog.categoryAdmin" = [ "roles/datacatalog.tagTemplateViewer" = [
module.load-sa-df-0.iam_email local.groups_iam.data-analysts, local.groups_iam.data-engineers
] ]
"roles/storage.objectCreator" = [ "roles/datacatalog.viewer" = [
module.transf-sa-df-0.iam_email, local.groups_iam.data-analysts, local.groups_iam.data-engineers
] ]
"roles/storage.objectViewer" = [ "roles/storage.objectViewer" = [
module.transf-sa-df-0.iam_email, local.groups_iam.data-analysts, local.groups_iam.data-engineers
] ]
"roles/storage.objectAdmin" = [module.transf-sa-df-0.iam_email]
} }
dwh_services = concat(var.project_services, [ dwh_services = concat(var.project_services, [
"bigquery.googleapis.com", "bigquery.googleapis.com",
@ -82,12 +76,13 @@ locals {
module "dwh-lnd-project" { module "dwh-lnd-project" {
source = "../../../modules/project" source = "../../../modules/project"
parent = var.folder_id parent = var.project_config.parent
billing_account = var.billing_account_id billing_account = var.project_config.billing_account_id
prefix = var.prefix project_create = var.project_config.billing_account_id != null
name = "dwh-lnd${local.project_suffix}" prefix = var.project_config.billing_account_id == null ? null : var.prefix
group_iam = local.dwh_group_iam name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}"
iam = local.dwh_lnd_iam iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {}
services = local.dwh_services services = local.dwh_services
service_encryption_key_ids = { service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)] bq = [try(local.service_encryption_keys.bq, null)]
@ -97,12 +92,13 @@ module "dwh-lnd-project" {
module "dwh-cur-project" { module "dwh-cur-project" {
source = "../../../modules/project" source = "../../../modules/project"
parent = var.folder_id parent = var.project_config.parent
billing_account = var.billing_account_id billing_account = var.project_config.billing_account_id
prefix = var.prefix project_create = var.project_config.billing_account_id != null
name = "dwh-cur${local.project_suffix}" prefix = var.project_config.billing_account_id == null ? null : var.prefix
group_iam = local.dwh_group_iam name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}"
iam = local.dwh_iam iam = var.project_config.billing_account_id != null ? local.dwh_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {}
services = local.dwh_services services = local.dwh_services
service_encryption_key_ids = { service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)] bq = [try(local.service_encryption_keys.bq, null)]
@ -112,12 +108,13 @@ module "dwh-cur-project" {
module "dwh-conf-project" { module "dwh-conf-project" {
source = "../../../modules/project" source = "../../../modules/project"
parent = var.folder_id parent = var.project_config.parent
billing_account = var.billing_account_id billing_account = var.project_config.billing_account_id
prefix = var.prefix project_create = var.project_config.billing_account_id != null
name = "dwh-conf${local.project_suffix}" prefix = var.project_config.billing_account_id == null ? null : var.prefix
group_iam = local.dwh_group_iam name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}"
iam = local.dwh_iam iam = var.project_config.billing_account_id != null ? local.dwh_iam : null
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null
services = local.dwh_services services = local.dwh_services
service_encryption_key_ids = { service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)] bq = [try(local.service_encryption_keys.bq, null)]
@ -138,7 +135,7 @@ module "dwh-lnd-bq-0" {
module "dwh-cur-bq-0" { module "dwh-cur-bq-0" {
source = "../../../modules/bigquery-dataset" source = "../../../modules/bigquery-dataset"
project_id = module.dwh-cur-project.project_id project_id = module.dwh-cur-project.project_id
id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0" id = "${replace(var.prefix, "-", "_")}_dwh_cur_bq_0"
location = var.location location = var.location
encryption_key = try(local.service_encryption_keys.bq, null) encryption_key = try(local.service_encryption_keys.bq, null)
} }

43
blueprints/data-solutions/data-platform-foundations/06-common.tf
View File

@ -14,35 +14,22 @@
# tfdoc:file:description common project. # tfdoc:file:description common project.
module "common-project" { locals {
source = "../../../modules/project" iam_common = {
parent = var.folder_id "roles/dlp.admin" = [local.groups_iam.data-security]
billing_account = var.billing_account_id "roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers]
prefix = var.prefix "roles/dlp.reader" = [local.groups_iam.data-engineers]
name = "cmn${local.project_suffix}"
group_iam = {
(local.groups.data-analysts) = [
"roles/datacatalog.viewer",
]
(local.groups.data-engineers) = [
"roles/dlp.reader",
"roles/dlp.user",
"roles/dlp.estimatesAdmin",
]
(local.groups.data-security) = [
"roles/dlp.admin",
"roles/datacatalog.admin"
]
}
iam = {
"roles/dlp.user" = [ "roles/dlp.user" = [
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email module.transf-sa-df-0.iam_email,
local.groups_iam.data-engineers
] ]
"roles/datacatalog.admin" = [local.groups_iam.data-security]
"roles/datacatalog.viewer" = [ "roles/datacatalog.viewer" = [
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email module.transf-sa-bq-0.iam_email,
local.groups_iam.data-analysts
] ]
"roles/datacatalog.categoryFineGrainedReader" = [ "roles/datacatalog.categoryFineGrainedReader" = [
module.transf-sa-df-0.iam_email, module.transf-sa-df-0.iam_email,
@ -51,6 +38,16 @@ module "common-project" {
# local.groups_iam.data-analysts # local.groups_iam.data-analysts
] ]
} }
}
module "common-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_common : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null
services = concat(var.project_services, [ services = concat(var.project_services, [
"datacatalog.googleapis.com", "datacatalog.googleapis.com",
"dlp.googleapis.com", "dlp.googleapis.com",

9
blueprints/data-solutions/data-platform-foundations/07-exposure.tf
View File

@ -16,8 +16,9 @@
module "exp-project" { module "exp-project" {
source = "../../../modules/project" source = "../../../modules/project"
parent = var.folder_id parent = var.project_config.parent
billing_account = var.billing_account_id billing_account = var.project_config.billing_account_id
prefix = var.prefix project_create = var.project_config.billing_account_id != null
name = "exp${local.project_suffix}" prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.exp : "${var.project_config.project_ids.exp}${local.project_suffix}"
} }

29
blueprints/data-solutions/data-platform-foundations/IAM.md
View File

@ -17,51 +17,48 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
| members | roles | | members | roles |
|---|---| |---|---|
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) |
|<b>drp-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) | |<b>drp-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
|<b>drp-cs-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | |<b>drp-cs-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
|<b>drp-ps-0</b><br><small><i>serviceAccount</i></small>|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) | |<b>drp-ps-0</b><br><small><i>serviceAccount</i></small>|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) |
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>orc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |<b>orc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
## Project <i>dwh-conf</i> ## Project <i>dwh-conf</i>
| members | roles | | members | roles |
|---|---| |---|---|
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>| |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | |<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
## Project <i>dwh-cur</i> ## Project <i>dwh-cur</i>
| members | roles | | members | roles |
|---|---| |---|---|
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>| |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | |<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
## Project <i>dwh-lnd</i> ## Project <i>dwh-lnd</i>
| members | roles | | members | roles |
|---|---| |---|---|
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>| |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | |<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
|<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | |<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) | |<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) |
## Project <i>lod</i> ## Project <i>lod</i>
| members | roles | | members | roles |
|---|---| |---|---|
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/compute.viewer](https://cloud.google.com/iam/docs/understanding-roles#compute.viewer) <br>[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) <br>[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) | |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) |
|<b>SERVICE_IDENTITY_dataflow-service-producer-prod</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |<b>SERVICE_IDENTITY_dataflow-service-producer-prod</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>| |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
@ -71,7 +68,7 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
| members | roles | | members | roles |
|---|---| |---|---|
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin) <br>[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor) <br>[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin) <br>[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor) <br>[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>SERVICE_IDENTITY_gcp-sa-cloudbuild</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |<b>SERVICE_IDENTITY_gcp-sa-cloudbuild</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>| |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|

45
blueprints/data-solutions/data-platform-foundations/README.md
View File

@ -213,13 +213,15 @@ While this blueprint can be used as a standalone deployment, it can also be call
```hcl ```hcl
module "data-platform" { module "data-platform" {
source = "./fabric/blueprints/data-solutions/data-platform-foundations" source = "./fabric/blueprints/data-solutions/data-platform-foundations"
billing_account_id = var.billing_account_id
folder_id = var.folder_id
organization_domain = "example.com" organization_domain = "example.com"
prefix = "myprefix" project_config = {
billing_account_id = "123456-123456-123456"
parent = "folders/12345678"
}
prefix = "myprefix"
} }
# tftest modules=43 resources=297 # tftest modules=43 resources=278
``` ```
## Customizations ## Customizations
@ -233,6 +235,14 @@ To create Cloud Key Management keys in the Data Platform you can uncomment the C
To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level. To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level.
To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules. To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules.
### Project Configuration
The solution can be deployed by creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly.
When you rely on existing projects, the blueprint is designed to rely on different projects configuring IAM binding with an additive approach. For discovery or experimentation purposes, you may also configure `project_config.project_ids` to point different projects to one project with the granularity you need. For example, deploy resources from the 'load' project with resources in the 'transformation' project.
Once you have identified the required project granularity for your use case, we suggest adapting the terraform script accordingly and relying on authoritative IAM binding.
## Demo pipeline ## Demo pipeline
The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features. The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features.
@ -244,20 +254,19 @@ You can find examples in the `[demo](./demo)` folder.
| name | description | type | required | default | | name | description | type | required | default |
|---|---|:---:|:---:|:---:| |---|---|:---:|:---:|:---:|
| [billing_account_id](variables.tf#L17) | Billing account id. | <code>string</code> | ✓ | | | [organization_domain](variables.tf#L156) | Organization domain. | <code>string</code> | ✓ | |
| [folder_id](variables.tf#L122) | Folder to be used for the networking resources in folders/nnnn format. | <code>string</code> | ✓ | | | [prefix](variables.tf#L161) | Prefix used for resource names. | <code>string</code> | ✓ | |
| [organization_domain](variables.tf#L166) | Organization domain. | <code>string</code> | ✓ | | | [project_config](variables.tf#L170) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10; billing_account_id &#61; optional&#40;string, null&#41;&#10; parent &#61; string&#10; project_ids &#61; optional&#40;object&#40;&#123;&#10; drop &#61; string&#10; load &#61; string&#10; orc &#61; string&#10; trf &#61; string&#10; dwh-lnd &#61; string&#10; dwh-cur &#61; string&#10; dwh-conf &#61; string&#10; common &#61; string&#10; exp &#61; string&#10; &#125;&#41;, &#123;&#10; drop &#61; &#34;drp&#34;&#10; load &#61; &#34;lod&#34;&#10; orc &#61; &#34;orc&#34;&#10; trf &#61; &#34;trf&#34;&#10; dwh-lnd &#61; &#34;dwh-lnd&#34;&#10; dwh-cur &#61; &#34;dwh-cur&#34;&#10; dwh-conf &#61; &#34;dwh-conf&#34;&#10; common &#61; &#34;cmn&#34;&#10; exp &#61; &#34;exp&#34;&#10; &#125;&#10; &#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | |
| [prefix](variables.tf#L171) | Prefix used for resource names. | <code>string</code> | ✓ | | | [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object&#40;&#123;&#10; disable_deployment &#61; optional&#40;bool&#41;&#10; environment_size &#61; optional&#40;string, &#34;ENVIRONMENT_SIZE_SMALL&#34;&#41;&#10; software_config &#61; optional&#40;object&#40;&#123;&#10; airflow_config_overrides &#61; optional&#40;any&#41;&#10; pypi_packages &#61; optional&#40;any&#41;&#10; env_variables &#61; optional&#40;map&#40;string&#41;&#41;&#10; image_version &#61; string&#10; &#125;&#41;, &#123;&#10; image_version &#61; &#34;composer-2-airflow-2&#34;&#10; &#125;&#41;&#10; workloads_config &#61; optional&#40;object&#40;&#123;&#10; scheduler &#61; optional&#40;object&#40;&#10; &#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; count &#61; number&#10; &#125;&#10; &#41;, &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; count &#61; 1&#10; &#125;&#41;&#10; web_server &#61; optional&#40;object&#40;&#10; &#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; &#125;&#10; &#41;, &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; &#125;&#41;&#10; worker &#61; optional&#40;object&#40;&#10; &#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; min_count &#61; number&#10; max_count &#61; number&#10; &#125;&#10; &#41;, &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; min_count &#61; 1&#10; max_count &#61; 3&#10; &#125;&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; environment_size &#61; &#34;ENVIRONMENT_SIZE_SMALL&#34;&#10; software_config &#61; &#123;&#10; image_version &#61; &#34;composer-2-airflow-2&#34;&#10; &#125;&#10; workloads_config &#61; &#123;&#10; scheduler &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; count &#61; 1&#10; &#125;&#10; web_server &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; &#125;&#10; worker &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; min_count &#61; 1&#10; max_count &#61; 3&#10; &#125;&#10; &#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [composer_config](variables.tf#L22) | Cloud Composer config. | <code title="object&#40;&#123;&#10; disable_deployment &#61; optional&#40;bool&#41;&#10; environment_size &#61; optional&#40;string, &#34;ENVIRONMENT_SIZE_SMALL&#34;&#41;&#10; software_config &#61; optional&#40;object&#40;&#123;&#10; airflow_config_overrides &#61; optional&#40;any&#41;&#10; pypi_packages &#61; optional&#40;any&#41;&#10; env_variables &#61; optional&#40;map&#40;string&#41;&#41;&#10; image_version &#61; string&#10; &#125;&#41;, &#123;&#10; image_version &#61; &#34;composer-2-airflow-2&#34;&#10; &#125;&#41;&#10; workloads_config &#61; optional&#40;object&#40;&#123;&#10; scheduler &#61; optional&#40;object&#40;&#10; &#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; count &#61; number&#10; &#125;&#10; &#41;, &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; count &#61; 1&#10; &#125;&#41;&#10; web_server &#61; optional&#40;object&#40;&#10; &#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; &#125;&#10; &#41;, &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; &#125;&#41;&#10; worker &#61; optional&#40;object&#40;&#10; &#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; min_count &#61; number&#10; max_count &#61; number&#10; &#125;&#10; &#41;, &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; min_count &#61; 1&#10; max_count &#61; 3&#10; &#125;&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; environment_size &#61; &#34;ENVIRONMENT_SIZE_SMALL&#34;&#10; software_config &#61; &#123;&#10; image_version &#61; &#34;composer-2-airflow-2&#34;&#10; &#125;&#10; workloads_config &#61; &#123;&#10; scheduler &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; count &#61; 1&#10; &#125;&#10; web_server &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; &#125;&#10; worker &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; min_count &#61; 1&#10; max_count &#61; 3&#10; &#125;&#10; &#125;&#10;&#125;">&#123;&#8230;&#125;</code> | | [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map&#40;map&#40;list&#40;string&#41;&#41;&#41;</code> | | <code title="&#123;&#10; &#34;3_Confidential&#34; &#61; null&#10; &#34;2_Private&#34; &#61; null&#10; &#34;1_Sensitive&#34; &#61; null&#10;&#125;">&#123;&#8230;&#125;</code> |
| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map&#40;map&#40;list&#40;string&#41;&#41;&#41;</code> | | <code title="&#123;&#10; &#34;3_Confidential&#34; &#61; null&#10; &#34;2_Private&#34; &#61; null&#10; &#34;1_Sensitive&#34; &#61; null&#10;&#125;">&#123;&#8230;&#125;</code> | | [data_force_destroy](variables.tf#L111) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
| [data_force_destroy](variables.tf#L116) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> | | [groups](variables.tf#L117) | User groups. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; data-analysts &#61; &#34;gcp-data-analysts&#34;&#10; data-engineers &#61; &#34;gcp-data-engineers&#34;&#10; data-security &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [groups](variables.tf#L127) | User groups. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; data-analysts &#61; &#34;gcp-data-analysts&#34;&#10; data-engineers &#61; &#34;gcp-data-engineers&#34;&#10; data-security &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | | [location](variables.tf#L127) | Location used for multi-regional resources. | <code>string</code> | | <code>&#34;eu&#34;</code> |
| [location](variables.tf#L137) | Location used for multi-regional resources. | <code>string</code> | | <code>&#34;eu&#34;</code> | | [network_config](variables.tf#L133) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10; host_project &#61; string&#10; network_self_link &#61; string&#10; subnet_self_links &#61; object&#40;&#123;&#10; load &#61; string&#10; transformation &#61; string&#10; orchestration &#61; string&#10; &#125;&#41;&#10; composer_ip_ranges &#61; object&#40;&#123;&#10; cloudsql &#61; string&#10; gke_master &#61; string&#10; &#125;&#41;&#10; composer_secondary_ranges &#61; object&#40;&#123;&#10; pods &#61; string&#10; services &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10; host_project &#61; string&#10; network_self_link &#61; string&#10; subnet_self_links &#61; object&#40;&#123;&#10; load &#61; string&#10; transformation &#61; string&#10; orchestration &#61; string&#10; &#125;&#41;&#10; composer_ip_ranges &#61; object&#40;&#123;&#10; cloudsql &#61; string&#10; gke_master &#61; string&#10; &#125;&#41;&#10; composer_secondary_ranges &#61; object&#40;&#123;&#10; pods &#61; string&#10; services &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | | [project_services](variables.tf#L204) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> | | <code title="&#91;&#10; &#34;cloudresourcemanager.googleapis.com&#34;,&#10; &#34;iam.googleapis.com&#34;,&#10; &#34;serviceusage.googleapis.com&#34;,&#10; &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |
| [project_services](variables.tf#L180) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> | | <code title="&#91;&#10; &#34;cloudresourcemanager.googleapis.com&#34;,&#10; &#34;iam.googleapis.com&#34;,&#10; &#34;serviceusage.googleapis.com&#34;,&#10; &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> | | [project_suffix](variables.tf#L215) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
| [project_suffix](variables.tf#L191) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> | | [region](variables.tf#L221) | Region used for regional resources. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| [region](variables.tf#L197) | Region used for regional resources. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> | | [service_encryption_keys](variables.tf#L227) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10; bq &#61; string&#10; composer &#61; string&#10; dataflow &#61; string&#10; storage &#61; string&#10; pubsub &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [service_encryption_keys](variables.tf#L203) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10; bq &#61; string&#10; composer &#61; string&#10; dataflow &#61; string&#10; storage &#61; string&#10; pubsub &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
## Outputs ## Outputs

8
blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
View File

@ -123,7 +123,7 @@ with models.DAG(
task_id="upsert_table_customers", task_id="upsert_table_customers",
project_id=DWH_LAND_PRJ, project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET, dataset_id=DWH_LAND_BQ_DATASET,
impersonation_chain=[TRF_SA_DF], impersonation_chain=[LOD_SA_DF],
table_resource={ table_resource={
"tableReference": {"tableId": "customers"}, "tableReference": {"tableId": "customers"},
}, },
@ -133,7 +133,7 @@ with models.DAG(
task_id="upsert_table_purchases", task_id="upsert_table_purchases",
project_id=DWH_LAND_PRJ, project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET, dataset_id=DWH_LAND_BQ_DATASET,
impersonation_chain=[TRF_SA_BQ], impersonation_chain=[LOD_SA_DF],
table_resource={ table_resource={
"tableReference": {"tableId": "purchases"} "tableReference": {"tableId": "purchases"}
}, },
@ -167,7 +167,7 @@ with models.DAG(
project_id=DWH_LAND_PRJ, project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET, dataset_id=DWH_LAND_BQ_DATASET,
table_id="customers", table_id="customers",
impersonation_chain=[TRF_SA_BQ], impersonation_chain=[LOD_SA_DF],
include_policy_tags=True, include_policy_tags=True,
schema_fields_updates=[ schema_fields_updates=[
{ "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" }, { "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" },
@ -182,7 +182,7 @@ with models.DAG(
project_id=DWH_LAND_PRJ, project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET, dataset_id=DWH_LAND_BQ_DATASET,
table_id="purchases", table_id="purchases",
impersonation_chain=[TRF_SA_BQ], impersonation_chain=[LOD_SA_DF],
include_policy_tags=True, include_policy_tags=True,
schema_fields_updates=[ schema_fields_updates=[
{ "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" }, { "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" },

4
blueprints/data-solutions/data-platform-foundations/demo/delete_table.py
View File

@ -122,13 +122,13 @@ with models.DAG(
delete_table_customers = BigQueryDeleteTableOperator( delete_table_customers = BigQueryDeleteTableOperator(
task_id="delete_table_customers", task_id="delete_table_customers",
deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers", deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers",
impersonation_chain=[TRF_SA_DF] impersonation_chain=[LOD_SA_DF]
) )
delete_table_purchases = BigQueryDeleteTableOperator( delete_table_purchases = BigQueryDeleteTableOperator(
task_id="delete_table_purchases", task_id="delete_table_purchases",
deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases", deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases",
impersonation_chain=[TRF_SA_DF] impersonation_chain=[LOD_SA_DF]
) )
delete_table_customer_purchase_curated = BigQueryDeleteTableOperator( delete_table_customer_purchase_curated = BigQueryDeleteTableOperator(

44
blueprints/data-solutions/data-platform-foundations/variables.tf
View File

@ -14,11 +14,6 @@
# tfdoc:file:description Terraform Variables. # tfdoc:file:description Terraform Variables.
variable "billing_account_id" {
description = "Billing account id."
type = string
}
variable "composer_config" { variable "composer_config" {
description = "Cloud Composer config." description = "Cloud Composer config."
type = object({ type = object({
@ -119,11 +114,6 @@ variable "data_force_destroy" {
default = false default = false
} }
variable "folder_id" {
description = "Folder to be used for the networking resources in folders/nnnn format."
type = string
}
variable "groups" { variable "groups" {
description = "User groups." description = "User groups."
type = map(string) type = map(string)
@ -177,6 +167,40 @@ variable "prefix" {
} }
} }
variable "project_config" {
description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
type = object({
billing_account_id = optional(string, null)
parent = string
project_ids = optional(object({
drop = string
load = string
orc = string
trf = string
dwh-lnd = string
dwh-cur = string
dwh-conf = string
common = string
exp = string
}), {
drop = "drp"
load = "lod"
orc = "orc"
trf = "trf"
dwh-lnd = "dwh-lnd"
dwh-cur = "dwh-cur"
dwh-conf = "dwh-conf"
common = "cmn"
exp = "exp"
}
)
})
validation {
condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null
error_message = "At least one attribute should be set."
}
}
variable "project_services" { variable "project_services" {
description = "List of core services enabled on all projects." description = "List of core services enabled on all projects."
type = list(string) type = list(string)

10
fast/stages/3-data-platform/dev/main.tf
View File

@ -18,13 +18,15 @@
module "data-platform" { module "data-platform" {
source = "../../../../blueprints/data-solutions/data-platform-foundations" source = "../../../../blueprints/data-solutions/data-platform-foundations"
billing_account_id = var.billing_account.id
composer_config = var.composer_config composer_config = var.composer_config
data_force_destroy = var.data_force_destroy data_force_destroy = var.data_force_destroy
data_catalog_tags = var.data_catalog_tags data_catalog_tags = var.data_catalog_tags
folder_id = var.folder_ids.data-platform-dev project_config = {
groups = var.groups billing_account_id = var.billing_account.id
location = var.location parent = var.folder_ids.data-platform-dev
}
groups = var.groups
location = var.location
network_config = { network_config = {
host_project = var.host_project_ids.dev-spoke-0 host_project = var.host_project_ids.dev-spoke-0
network_self_link = var.vpc_self_links.dev-spoke-0 network_self_link = var.vpc_self_links.dev-spoke-0

8
tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf
View File

@ -17,7 +17,9 @@
module "test" { module "test" {
source = "../../../../../blueprints/data-solutions/data-platform-foundations/" source = "../../../../../blueprints/data-solutions/data-platform-foundations/"
organization_domain = "example.com" organization_domain = "example.com"
billing_account_id = "123456-123456-123456" project_config = {
folder_id = "folders/12345678" billing_account_id = "123456-123456-123456"
prefix = "prefix" parent = "folders/12345678"
}
prefix = "prefix"
} }

2
tests/blueprints/data_solutions/data_platform_foundations/test_plan.py
View File

@ -23,4 +23,4 @@ def test_resources(e2e_plan_runner):
modules, resources = e2e_plan_runner(FIXTURES_DIR) modules, resources = e2e_plan_runner(FIXTURES_DIR)
assert len(modules) == 42 assert len(modules) == 42
assert len(resources) == 296 assert len(resources) == 277