Merge pull request #1165 from GoogleCloudPlatform/lcaggio/dp-projectcreate
DataPlatform: Support project creation
This commit is contained in:
lcaggio
GitHub
commit
89da766801
|
|
@ -15,36 +15,34 @@
|
|||
# tfdoc:file:description drop off project and resources.
|
||||
|
||||
locals {
|
||||
drop_orch_service_accounts = [
|
||||
module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
iam_drp = {
|
||||
"roles/bigquery.dataEditor" = [
|
||||
module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/bigquery.user" = [
|
||||
module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
|
||||
"roles/pubsub.subscriber" = [
|
||||
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
|
||||
]
|
||||
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
|
||||
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
|
||||
"roles/storage.objectAdmin" = [
|
||||
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "drop-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "drp${local.project_suffix}"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/bigquery.dataEditor",
|
||||
"roles/pubsub.editor",
|
||||
"roles/storage.admin",
|
||||
]
|
||||
}
|
||||
iam = {
|
||||
"roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
|
||||
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
|
||||
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
|
||||
"roles/pubsub.subscriber" = concat(
|
||||
local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email]
|
||||
)
|
||||
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
|
||||
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
|
||||
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
|
||||
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
|
||||
}
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
|
||||
iam = var.project_config.billing_account_id != null ? local.iam_drp : null
|
||||
iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,19 @@
|
|||
# tfdoc:file:description Load project and VPC.
|
||||
|
||||
locals {
|
||||
iam_load = {
|
||||
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
|
||||
"roles/dataflow.admin" = [
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
module.load-sa-df-0.iam_email,
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/dataflow.developer" = [
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
|
||||
"roles/storage.objectAdmin" = local.load_service_accounts
|
||||
}
|
||||
load_service_accounts = [
|
||||
"serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
|
||||
module.load-sa-df-0.iam_email
|
||||
|
|
@ -35,26 +48,13 @@ locals {
|
|||
|
||||
module "load-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "lod${local.project_suffix}"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/compute.viewer",
|
||||
"roles/dataflow.admin",
|
||||
"roles/dataflow.developer",
|
||||
"roles/viewer",
|
||||
]
|
||||
}
|
||||
iam = {
|
||||
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
|
||||
"roles/dataflow.admin" = [
|
||||
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
|
||||
]
|
||||
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
|
||||
"roles/storage.objectAdmin" = local.load_service_accounts
|
||||
}
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}"
|
||||
iam = var.project_config.billing_account_id != null ? local.iam_load : null
|
||||
iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
|
|
@ -86,8 +86,13 @@ module "load-sa-df-0" {
|
|||
name = "load-df-0"
|
||||
display_name = "Data platform Dataflow load service account"
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
|
||||
"roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers,
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
],
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -107,11 +112,11 @@ module "load-vpc" {
|
|||
source = "../../../modules/net-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.load-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
name = "${var.prefix}-lod"
|
||||
subnets = [
|
||||
{
|
||||
ip_cidr_range = "10.10.0.0/24"
|
||||
name = "default"
|
||||
name = "${var.prefix}-lod"
|
||||
region = var.region
|
||||
}
|
||||
]
|
||||
|
|
@ -131,7 +136,7 @@ module "load-nat" {
|
|||
source = "../../../modules/net-cloudnat"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.load-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
name = "${var.prefix}-lod"
|
||||
region = var.region
|
||||
router_network = module.load-vpc.0.name
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,42 @@
|
|||
# tfdoc:file:description Orchestration project and VPC.
|
||||
|
||||
locals {
|
||||
iam_orch = {
|
||||
"roles/artifactregistry.admin" = [local.groups_iam.data-engineers]
|
||||
"roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email]
|
||||
"roles/bigquery.dataEditor" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers]
|
||||
"roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email]
|
||||
"roles/composer.admin" = [local.groups_iam.data-engineers]
|
||||
"roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers]
|
||||
"roles/composer.ServiceAgentV2Ext" = [
|
||||
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
|
||||
]
|
||||
"roles/composer.worker" = [
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers]
|
||||
"roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers]
|
||||
"roles/storage.objectAdmin" = [
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
module.orch-sa-df-build.iam_email,
|
||||
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
|
||||
"serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
|
||||
}
|
||||
orch_subnet = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.subnet_self_links.orchestration
|
||||
|
|
@ -34,57 +70,14 @@ locals {
|
|||
|
||||
module "orch-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "orc${local.project_suffix}"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/bigquery.dataEditor",
|
||||
"roles/bigquery.jobUser",
|
||||
"roles/cloudbuild.builds.editor",
|
||||
"roles/composer.admin",
|
||||
"roles/composer.environmentAndStorageObjectAdmin",
|
||||
"roles/iap.httpsResourceAccessor",
|
||||
"roles/iam.serviceAccountUser",
|
||||
"roles/storage.objectAdmin",
|
||||
"roles/storage.admin",
|
||||
"roles/artifactregistry.admin",
|
||||
"roles/serviceusage.serviceUsageConsumer",
|
||||
]
|
||||
}
|
||||
iam = {
|
||||
"roles/bigquery.dataEditor" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
]
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/composer.ServiceAgentV2Ext" = [
|
||||
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
|
||||
]
|
||||
"roles/composer.worker" = [
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/storage.objectAdmin" = [
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
module.orch-sa-df-build.iam_email,
|
||||
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
|
||||
"serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
|
||||
]
|
||||
"roles/artifactregistry.reader" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
]
|
||||
"roles/cloudbuild.serviceAgent" = [
|
||||
module.orch-sa-df-build.iam_email,
|
||||
]
|
||||
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
|
||||
}
|
||||
oslogin = false
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
|
||||
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
|
||||
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
|
||||
oslogin = false
|
||||
services = concat(var.project_services, [
|
||||
"artifactregistry.googleapis.com",
|
||||
"bigquery.googleapis.com",
|
||||
|
|
@ -132,11 +125,11 @@ module "orch-vpc" {
|
|||
source = "../../../modules/net-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.orch-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
name = "${var.prefix}-orch"
|
||||
subnets = [
|
||||
{
|
||||
ip_cidr_range = "10.10.0.0/24"
|
||||
name = "default"
|
||||
name = "${var.prefix}-orch"
|
||||
region = var.region
|
||||
secondary_ip_ranges = {
|
||||
pods = "10.10.8.0/22"
|
||||
|
|
@ -160,7 +153,7 @@ module "orch-nat" {
|
|||
count = local.use_shared_vpc ? 0 : 1
|
||||
source = "../../../modules/net-cloudnat"
|
||||
project_id = module.orch-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
name = "${var.prefix}-orch"
|
||||
region = var.region
|
||||
router_network = module.orch-vpc.0.name
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,19 @@
|
|||
# tfdoc:file:description Trasformation project and VPC.
|
||||
|
||||
locals {
|
||||
iam_trf = {
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/dataflow.admin" = [
|
||||
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/dataflow.worker" = [module.transf-sa-df-0.iam_email]
|
||||
"roles/storage.objectAdmin" = [
|
||||
module.transf-sa-df-0.iam_email,
|
||||
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
|
||||
]
|
||||
}
|
||||
transf_subnet = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.subnet_self_links.orchestration
|
||||
|
|
@ -29,31 +42,13 @@ locals {
|
|||
|
||||
module "transf-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "trf${local.project_suffix}"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/bigquery.jobUser",
|
||||
"roles/dataflow.admin",
|
||||
]
|
||||
}
|
||||
iam = {
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
]
|
||||
"roles/dataflow.admin" = [
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/dataflow.worker" = [
|
||||
module.transf-sa-df-0.iam_email
|
||||
]
|
||||
"roles/storage.objectAdmin" = [
|
||||
module.transf-sa-df-0.iam_email,
|
||||
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
|
||||
]
|
||||
}
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
|
||||
iam = var.project_config.billing_account_id != null ? local.iam_trf : null
|
||||
iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
|
|
@ -131,11 +126,11 @@ module "transf-vpc" {
|
|||
source = "../../../modules/net-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.transf-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
name = "${var.prefix}-trf"
|
||||
subnets = [
|
||||
{
|
||||
ip_cidr_range = "10.10.0.0/24"
|
||||
name = "default"
|
||||
name = "${var.prefix}-trf"
|
||||
region = var.region
|
||||
}
|
||||
]
|
||||
|
|
@ -155,7 +150,7 @@ module "transf-nat" {
|
|||
source = "../../../modules/net-cloudnat"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.transf-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
name = "${var.prefix}-trf"
|
||||
region = var.region
|
||||
router_network = module.transf-vpc.0.name
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,54 +15,48 @@
|
|||
# tfdoc:file:description Data Warehouse projects.
|
||||
|
||||
locals {
|
||||
dwh_group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/bigquery.dataEditor",
|
||||
"roles/storage.admin",
|
||||
],
|
||||
(local.groups.data-analysts) = [
|
||||
"roles/bigquery.dataViewer",
|
||||
"roles/bigquery.jobUser",
|
||||
"roles/bigquery.metadataViewer",
|
||||
"roles/bigquery.user",
|
||||
"roles/datacatalog.viewer",
|
||||
"roles/datacatalog.tagTemplateViewer",
|
||||
"roles/storage.objectViewer",
|
||||
]
|
||||
}
|
||||
dwh_lnd_iam = {
|
||||
"roles/bigquery.dataOwner" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
]
|
||||
"roles/bigquery.dataViewer" = [
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
]
|
||||
"roles/datacatalog.categoryAdmin" = [
|
||||
module.transf-sa-bq-0.iam_email
|
||||
]
|
||||
"roles/storage.objectCreator" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/datacatalog.categoryAdmin" = [module.transf-sa-bq-0.iam_email]
|
||||
"roles/datacatalog.tagTemplateViewer" = [local.groups_iam.data-engineers]
|
||||
"roles/datacatalog.viewer" = [local.groups_iam.data-engineers]
|
||||
"roles/storage.objectCreator" = [module.load-sa-df-0.iam_email]
|
||||
"roles/storage.objectViewer" = [local.groups_iam.data-engineers]
|
||||
}
|
||||
dwh_iam = {
|
||||
"roles/bigquery.dataOwner" = [
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
]
|
||||
"roles/bigquery.dataViewer" = [
|
||||
local.groups_iam.data-analysts,
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
local.groups_iam.data-analysts,
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/datacatalog.categoryAdmin" = [
|
||||
module.load-sa-df-0.iam_email
|
||||
"roles/datacatalog.tagTemplateViewer" = [
|
||||
local.groups_iam.data-analysts, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/storage.objectCreator" = [
|
||||
module.transf-sa-df-0.iam_email,
|
||||
"roles/datacatalog.viewer" = [
|
||||
local.groups_iam.data-analysts, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/storage.objectViewer" = [
|
||||
module.transf-sa-df-0.iam_email,
|
||||
local.groups_iam.data-analysts, local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/storage.objectAdmin" = [module.transf-sa-df-0.iam_email]
|
||||
}
|
||||
dwh_services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
|
|
@ -82,12 +76,13 @@ locals {
|
|||
|
||||
module "dwh-lnd-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "dwh-lnd${local.project_suffix}"
|
||||
group_iam = local.dwh_group_iam
|
||||
iam = local.dwh_lnd_iam
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}"
|
||||
iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {}
|
||||
iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {}
|
||||
services = local.dwh_services
|
||||
service_encryption_key_ids = {
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
|
|
@ -97,12 +92,13 @@ module "dwh-lnd-project" {
|
|||
|
||||
module "dwh-cur-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "dwh-cur${local.project_suffix}"
|
||||
group_iam = local.dwh_group_iam
|
||||
iam = local.dwh_iam
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}"
|
||||
iam = var.project_config.billing_account_id != null ? local.dwh_iam : {}
|
||||
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {}
|
||||
services = local.dwh_services
|
||||
service_encryption_key_ids = {
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
|
|
@ -112,12 +108,13 @@ module "dwh-cur-project" {
|
|||
|
||||
module "dwh-conf-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "dwh-conf${local.project_suffix}"
|
||||
group_iam = local.dwh_group_iam
|
||||
iam = local.dwh_iam
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}"
|
||||
iam = var.project_config.billing_account_id != null ? local.dwh_iam : null
|
||||
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null
|
||||
services = local.dwh_services
|
||||
service_encryption_key_ids = {
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
|
|
@ -138,7 +135,7 @@ module "dwh-lnd-bq-0" {
|
|||
module "dwh-cur-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.dwh-cur-project.project_id
|
||||
id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0"
|
||||
id = "${replace(var.prefix, "-", "_")}_dwh_cur_bq_0"
|
||||
location = var.location
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,35 +14,22 @@
|
|||
|
||||
# tfdoc:file:description common project.
|
||||
|
||||
module "common-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "cmn${local.project_suffix}"
|
||||
group_iam = {
|
||||
(local.groups.data-analysts) = [
|
||||
"roles/datacatalog.viewer",
|
||||
]
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/dlp.reader",
|
||||
"roles/dlp.user",
|
||||
"roles/dlp.estimatesAdmin",
|
||||
]
|
||||
(local.groups.data-security) = [
|
||||
"roles/dlp.admin",
|
||||
"roles/datacatalog.admin"
|
||||
]
|
||||
}
|
||||
iam = {
|
||||
locals {
|
||||
iam_common = {
|
||||
"roles/dlp.admin" = [local.groups_iam.data-security]
|
||||
"roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers]
|
||||
"roles/dlp.reader" = [local.groups_iam.data-engineers]
|
||||
"roles/dlp.user" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email
|
||||
module.transf-sa-df-0.iam_email,
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
"roles/datacatalog.admin" = [local.groups_iam.data-security]
|
||||
"roles/datacatalog.viewer" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.transf-sa-bq-0.iam_email
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
local.groups_iam.data-analysts
|
||||
]
|
||||
"roles/datacatalog.categoryFineGrainedReader" = [
|
||||
module.transf-sa-df-0.iam_email,
|
||||
|
|
@ -51,6 +38,16 @@ module "common-project" {
|
|||
# local.groups_iam.data-analysts
|
||||
]
|
||||
}
|
||||
}
|
||||
module "common-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}"
|
||||
iam = var.project_config.billing_account_id != null ? local.iam_common : null
|
||||
iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null
|
||||
services = concat(var.project_services, [
|
||||
"datacatalog.googleapis.com",
|
||||
"dlp.googleapis.com",
|
||||
|
|
|
|||
|
|
@ -16,8 +16,9 @@
|
|||
|
||||
module "exp-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "exp${local.project_suffix}"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
prefix = var.project_config.billing_account_id == null ? null : var.prefix
|
||||
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.exp : "${var.project_config.project_ids.exp}${local.project_suffix}"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,51 +17,48 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
|
|||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) |
|
||||
|<b>drp-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
|
||||
|<b>drp-cs-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
|
||||
|<b>drp-ps-0</b><br><small><i>serviceAccount</i></small>|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) |
|
||||
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|<b>orc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|
||||
## Project <i>dwh-conf</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
||||
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
||||
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|
||||
|<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
|
||||
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|
||||
## Project <i>dwh-cur</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
||||
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
||||
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|
||||
|<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
|
||||
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|
||||
## Project <i>dwh-lnd</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|
||||
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
||||
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
|
||||
|<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|
||||
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) |
|
||||
|<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|
||||
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) |
|
||||
|
||||
## Project <i>lod</i>
|
||||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/compute.viewer](https://cloud.google.com/iam/docs/understanding-roles#compute.viewer) <br>[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) <br>[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) |
|
||||
|<b>SERVICE_IDENTITY_dataflow-service-producer-prod</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
||||
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|
|
@ -71,7 +68,7 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
|
|||
|
||||
| members | roles |
|
||||
|---|---|
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin) <br>[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor) <br>[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin) <br>[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor) <br>[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|<b>SERVICE_IDENTITY_gcp-sa-cloudbuild</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|
||||
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|
||||
|
|
|
|||
|
|
@ -213,13 +213,15 @@ While this blueprint can be used as a standalone deployment, it can also be call
|
|||
```hcl
|
||||
module "data-platform" {
|
||||
source = "./fabric/blueprints/data-solutions/data-platform-foundations"
|
||||
billing_account_id = var.billing_account_id
|
||||
folder_id = var.folder_id
|
||||
organization_domain = "example.com"
|
||||
prefix = "myprefix"
|
||||
project_config = {
|
||||
billing_account_id = "123456-123456-123456"
|
||||
parent = "folders/12345678"
|
||||
}
|
||||
prefix = "myprefix"
|
||||
}
|
||||
|
||||
# tftest modules=43 resources=297
|
||||
# tftest modules=43 resources=278
|
||||
```
|
||||
|
||||
## Customizations
|
||||
|
|
@ -233,6 +235,14 @@ To create Cloud Key Management keys in the Data Platform you can uncomment the C
|
|||
To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level.
|
||||
To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules.
|
||||
|
||||
### Project Configuration
|
||||
|
||||
The solution can be deployed by creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly.
|
||||
|
||||
When you rely on existing projects, the blueprint is designed to rely on different projects configuring IAM binding with an additive approach. For discovery or experimentation purposes, you may also configure `project_config.project_ids` to point different projects to one project with the granularity you need. For example, deploy resources from the 'load' project with resources in the 'transformation' project.
|
||||
|
||||
Once you have identified the required project granularity for your use case, we suggest adapting the terraform script accordingly and relying on authoritative IAM binding.
|
||||
|
||||
## Demo pipeline
|
||||
|
||||
The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features.
|
||||
|
|
@ -244,20 +254,19 @@ You can find examples in the `[demo](./demo)` folder.
|
|||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [billing_account_id](variables.tf#L17) | Billing account id. | <code>string</code> | ✓ | |
|
||||
| [folder_id](variables.tf#L122) | Folder to be used for the networking resources in folders/nnnn format. | <code>string</code> | ✓ | |
|
||||
| [organization_domain](variables.tf#L166) | Organization domain. | <code>string</code> | ✓ | |
|
||||
| [prefix](variables.tf#L171) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||
| [composer_config](variables.tf#L22) | Cloud Composer config. | <code title="object({ disable_deployment = optional(bool) environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional(object({ airflow_config_overrides = optional(any) pypi_packages = optional(any) env_variables = optional(map(string)) image_version = string }), { image_version = "composer-2-airflow-2" }) workloads_config = optional(object({ scheduler = optional(object( { cpu = number memory_gb = number storage_gb = number count = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 }) web_server = optional(object( { cpu = number memory_gb = number storage_gb = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 }) worker = optional(object( { cpu = number memory_gb = number storage_gb = number min_count = number max_count = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 }) })) })">object({…})</code> | | <code title="{ environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" } workloads_config = { scheduler = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 } web_server = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 } worker = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 } } }">{…}</code> |
|
||||
| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code title="{ "3_Confidential" = null "2_Private" = null "1_Sensitive" = null }">{…}</code> |
|
||||
| [data_force_destroy](variables.tf#L116) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
||||
| [groups](variables.tf#L127) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
||||
| [location](variables.tf#L137) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
||||
| [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string network_self_link = string subnet_self_links = object({ load = string transformation = string orchestration = string }) composer_ip_ranges = object({ cloudsql = string gke_master = string }) composer_secondary_ranges = object({ pods = string services = string }) })">object({…})</code> | | <code>null</code> |
|
||||
| [project_services](variables.tf#L180) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> |
|
||||
| [project_suffix](variables.tf#L191) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L197) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [service_encryption_keys](variables.tf#L203) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = string composer = string dataflow = string storage = string pubsub = string })">object({…})</code> | | <code>null</code> |
|
||||
| [organization_domain](variables.tf#L156) | Organization domain. | <code>string</code> | ✓ | |
|
||||
| [prefix](variables.tf#L161) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||
| [project_config](variables.tf#L170) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) parent = string project_ids = optional(object({ drop = string load = string orc = string trf = string dwh-lnd = string dwh-cur = string dwh-conf = string common = string exp = string }), { drop = "drp" load = "lod" orc = "orc" trf = "trf" dwh-lnd = "dwh-lnd" dwh-cur = "dwh-cur" dwh-conf = "dwh-conf" common = "cmn" exp = "exp" } ) })">object({…})</code> | ✓ | |
|
||||
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object({ disable_deployment = optional(bool) environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional(object({ airflow_config_overrides = optional(any) pypi_packages = optional(any) env_variables = optional(map(string)) image_version = string }), { image_version = "composer-2-airflow-2" }) workloads_config = optional(object({ scheduler = optional(object( { cpu = number memory_gb = number storage_gb = number count = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 }) web_server = optional(object( { cpu = number memory_gb = number storage_gb = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 }) worker = optional(object( { cpu = number memory_gb = number storage_gb = number min_count = number max_count = number } ), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 }) })) })">object({…})</code> | | <code title="{ environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" } workloads_config = { scheduler = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 } web_server = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 } worker = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 } } }">{…}</code> |
|
||||
| [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code title="{ "3_Confidential" = null "2_Private" = null "1_Sensitive" = null }">{…}</code> |
|
||||
| [data_force_destroy](variables.tf#L111) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
||||
| [groups](variables.tf#L117) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
||||
| [location](variables.tf#L127) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
||||
| [network_config](variables.tf#L133) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string network_self_link = string subnet_self_links = object({ load = string transformation = string orchestration = string }) composer_ip_ranges = object({ cloudsql = string gke_master = string }) composer_secondary_ranges = object({ pods = string services = string }) })">object({…})</code> | | <code>null</code> |
|
||||
| [project_services](variables.tf#L204) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> |
|
||||
| [project_suffix](variables.tf#L215) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L221) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [service_encryption_keys](variables.tf#L227) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = string composer = string dataflow = string storage = string pubsub = string })">object({…})</code> | | <code>null</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ with models.DAG(
|
|||
task_id="upsert_table_customers",
|
||||
project_id=DWH_LAND_PRJ,
|
||||
dataset_id=DWH_LAND_BQ_DATASET,
|
||||
impersonation_chain=[TRF_SA_DF],
|
||||
impersonation_chain=[LOD_SA_DF],
|
||||
table_resource={
|
||||
"tableReference": {"tableId": "customers"},
|
||||
},
|
||||
|
|
@ -133,7 +133,7 @@ with models.DAG(
|
|||
task_id="upsert_table_purchases",
|
||||
project_id=DWH_LAND_PRJ,
|
||||
dataset_id=DWH_LAND_BQ_DATASET,
|
||||
impersonation_chain=[TRF_SA_BQ],
|
||||
impersonation_chain=[LOD_SA_DF],
|
||||
table_resource={
|
||||
"tableReference": {"tableId": "purchases"}
|
||||
},
|
||||
|
|
@ -167,7 +167,7 @@ with models.DAG(
|
|||
project_id=DWH_LAND_PRJ,
|
||||
dataset_id=DWH_LAND_BQ_DATASET,
|
||||
table_id="customers",
|
||||
impersonation_chain=[TRF_SA_BQ],
|
||||
impersonation_chain=[LOD_SA_DF],
|
||||
include_policy_tags=True,
|
||||
schema_fields_updates=[
|
||||
{ "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" },
|
||||
|
|
@ -182,7 +182,7 @@ with models.DAG(
|
|||
project_id=DWH_LAND_PRJ,
|
||||
dataset_id=DWH_LAND_BQ_DATASET,
|
||||
table_id="purchases",
|
||||
impersonation_chain=[TRF_SA_BQ],
|
||||
impersonation_chain=[LOD_SA_DF],
|
||||
include_policy_tags=True,
|
||||
schema_fields_updates=[
|
||||
{ "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" },
|
||||
|
|
|
|||
|
|
@ -122,13 +122,13 @@ with models.DAG(
|
|||
delete_table_customers = BigQueryDeleteTableOperator(
|
||||
task_id="delete_table_customers",
|
||||
deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers",
|
||||
impersonation_chain=[TRF_SA_DF]
|
||||
impersonation_chain=[LOD_SA_DF]
|
||||
)
|
||||
|
||||
delete_table_purchases = BigQueryDeleteTableOperator(
|
||||
task_id="delete_table_purchases",
|
||||
deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases",
|
||||
impersonation_chain=[TRF_SA_DF]
|
||||
impersonation_chain=[LOD_SA_DF]
|
||||
)
|
||||
|
||||
delete_table_customer_purchase_curated = BigQueryDeleteTableOperator(
|
||||
|
|
|
|||
|
|
@ -14,11 +14,6 @@
|
|||
|
||||
# tfdoc:file:description Terraform Variables.
|
||||
|
||||
variable "billing_account_id" {
|
||||
description = "Billing account id."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "composer_config" {
|
||||
description = "Cloud Composer config."
|
||||
type = object({
|
||||
|
|
@ -119,11 +114,6 @@ variable "data_force_destroy" {
|
|||
default = false
|
||||
}
|
||||
|
||||
variable "folder_id" {
|
||||
description = "Folder to be used for the networking resources in folders/nnnn format."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "groups" {
|
||||
description = "User groups."
|
||||
type = map(string)
|
||||
|
|
@ -177,6 +167,40 @@ variable "prefix" {
|
|||
}
|
||||
}
|
||||
|
||||
variable "project_config" {
|
||||
description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
|
||||
type = object({
|
||||
billing_account_id = optional(string, null)
|
||||
parent = string
|
||||
project_ids = optional(object({
|
||||
drop = string
|
||||
load = string
|
||||
orc = string
|
||||
trf = string
|
||||
dwh-lnd = string
|
||||
dwh-cur = string
|
||||
dwh-conf = string
|
||||
common = string
|
||||
exp = string
|
||||
}), {
|
||||
drop = "drp"
|
||||
load = "lod"
|
||||
orc = "orc"
|
||||
trf = "trf"
|
||||
dwh-lnd = "dwh-lnd"
|
||||
dwh-cur = "dwh-cur"
|
||||
dwh-conf = "dwh-conf"
|
||||
common = "cmn"
|
||||
exp = "exp"
|
||||
}
|
||||
)
|
||||
})
|
||||
validation {
|
||||
condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null
|
||||
error_message = "At least one attribute should be set."
|
||||
}
|
||||
}
|
||||
|
||||
variable "project_services" {
|
||||
description = "List of core services enabled on all projects."
|
||||
type = list(string)
|
||||
|
|
|
|||
|
|
@ -18,13 +18,15 @@
|
|||
|
||||
module "data-platform" {
|
||||
source = "../../../../blueprints/data-solutions/data-platform-foundations"
|
||||
billing_account_id = var.billing_account.id
|
||||
composer_config = var.composer_config
|
||||
data_force_destroy = var.data_force_destroy
|
||||
data_catalog_tags = var.data_catalog_tags
|
||||
folder_id = var.folder_ids.data-platform-dev
|
||||
groups = var.groups
|
||||
location = var.location
|
||||
project_config = {
|
||||
billing_account_id = var.billing_account.id
|
||||
parent = var.folder_ids.data-platform-dev
|
||||
}
|
||||
groups = var.groups
|
||||
location = var.location
|
||||
network_config = {
|
||||
host_project = var.host_project_ids.dev-spoke-0
|
||||
network_self_link = var.vpc_self_links.dev-spoke-0
|
||||
|
|
|
|||
|
|
@ -17,7 +17,9 @@
|
|||
module "test" {
|
||||
source = "../../../../../blueprints/data-solutions/data-platform-foundations/"
|
||||
organization_domain = "example.com"
|
||||
billing_account_id = "123456-123456-123456"
|
||||
folder_id = "folders/12345678"
|
||||
prefix = "prefix"
|
||||
project_config = {
|
||||
billing_account_id = "123456-123456-123456"
|
||||
parent = "folders/12345678"
|
||||
}
|
||||
prefix = "prefix"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,4 +23,4 @@ def test_resources(e2e_plan_runner):
|
|||
modules, resources = e2e_plan_runner(FIXTURES_DIR)
|
||||
|
||||
assert len(modules) == 42
|
||||
assert len(resources) == 296
|
||||
assert len(resources) == 277
|
||||
|
|
|
|||
Loading…
Reference in New Issue