From ad6667a8fa658cbc8725e17436473fb7ab7357df Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sun, 19 Feb 2023 22:37:32 +0100 Subject: [PATCH 01/13] First commit --- .../data-platform-foundations/01-dropoff.tf | 7 ++-- .../data-platform-foundations/02-load.tf | 7 ++-- .../03-orchestration.tf | 7 ++-- .../04-transformation.tf | 7 ++-- .../05-datawarehouse.tf | 21 +++++++----- .../data-platform-foundations/06-common.tf | 7 ++-- .../data-platform-foundations/07-exposure.tf | 7 ++-- .../data-platform-foundations/variables.tf | 34 +++++++++++++++++++ 8 files changed, 70 insertions(+), 27 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index 177f940a..f1dc492e 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -23,9 +23,10 @@ locals { module "drop-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "drp${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" group_iam = { (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index 74cb9f8b..9c025e69 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -36,9 +36,10 @@ locals { module "load-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "lod${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}" group_iam = { (local.groups.data-engineers) = [ "roles/compute.viewer", diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index a202afdd..c7f59578 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -35,9 +35,10 @@ locals { module "orch-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "orc${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" group_iam = { (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index 3d3a818c..c2b9b1e2 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -30,9 +30,10 @@ locals { module "transf-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "trf${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" group_iam = { (local.groups.data-engineers) = [ "roles/bigquery.jobUser", diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index 0db5ce44..6bc1a28e 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -83,9 +83,10 @@ locals { module "dwh-lnd-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-lnd${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" group_iam = local.dwh_group_iam iam = local.dwh_lnd_iam services = local.dwh_services @@ -98,9 +99,10 @@ module "dwh-lnd-project" { module "dwh-cur-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-cur${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" group_iam = local.dwh_group_iam iam = local.dwh_iam services = local.dwh_services @@ -113,9 +115,10 @@ module "dwh-cur-project" { module "dwh-conf-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-conf${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" group_iam = local.dwh_group_iam iam = local.dwh_iam services = local.dwh_services diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 80451500..64df392b 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -17,9 +17,10 @@ module "common-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "cmn${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" group_iam = { (local.groups.data-analysts) = [ "roles/datacatalog.viewer", diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf index 030be0b8..4a8071c0 100644 --- a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf +++ b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf @@ -17,7 +17,8 @@ module "exp-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "exp${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.exp : "${var.project_config.project_ids.exp}${local.project_suffix}" } diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index 6c25406a..de033f72 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -177,6 +177,40 @@ variable "prefix" { } } +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = optional(string, null) + project_ids = optional(object({ + drop = string + load = string + orc = string + trf = string + dwh-lnd = string + dwh-cur = string + dwh-conf = string + common = string + exp = string + }), { + drop = "drp" + load = "lod" + orc = "orc" + trf = "trf" + dwh-lnd = "dwh-lnd" + dwh-cur = "dwh-cur" + dwh-conf = "dwh-conf" + common = "cmn" + exp = "exp" + } + ) + }) + default = {} + validation { + condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null + error_message = "At least one attribute should be set." + } +} + variable "project_services" { description = "List of core services enabled on all projects." type = list(string) From c8f25512eb5094cb9bcd73a28200a8e2cfe346ea Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 00:39:37 +0100 Subject: [PATCH 02/13] Fix variabler to support existing projects. --- .../data-platform-foundations/01-dropoff.tf | 34 +++++----- .../data-platform-foundations/02-load.tf | 43 +++++++------ .../03-orchestration.tf | 63 ++++++++++--------- .../04-transformation.tf | 51 ++++++++------- .../05-datawarehouse.tf | 29 +++++---- .../data-platform-foundations/06-common.tf | 23 ++++--- .../data-platform-foundations/07-exposure.tf | 2 +- .../data-platform-foundations/README.md | 4 ++ .../data-platform-foundations/variables.tf | 2 +- 9 files changed, 136 insertions(+), 115 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index f1dc492e..4c4264d3 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -15,37 +15,37 @@ # tfdoc:file:description drop off project and resources. locals { - drop_orch_service_accounts = [ - module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email - ] -} - -module "drop-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" - group_iam = { + group_iam_drp = { (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", "roles/pubsub.editor", "roles/storage.admin", ] } - iam = { + iam_drp = { "roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email] "roles/bigquery.user" = [module.load-sa-df-0.iam_email] "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] - "roles/pubsub.subscriber" = concat( - local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email] - ) + "roles/pubsub.subscriber" = [ + module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email + ] "roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email] "roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email] "roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email] "roles/storage.admin" = [module.load-sa-df-0.iam_email] } +} + +module "drop-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" + # group_iam = local.group_iam_drp + iam = var.project_config.billing_account_id != null ? local.iam_drp : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index 9c025e69..b547f050 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -15,6 +15,22 @@ # tfdoc:file:description Load project and VPC. locals { + group_iam_load = { + (local.groups.data-engineers) = [ + "roles/compute.viewer", + "roles/dataflow.admin", + "roles/dataflow.developer", + "roles/viewer", + ] + } + iam_load = { + "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] + "roles/dataflow.admin" = [ + module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email + ] + "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] + "roles/storage.objectAdmin" = local.load_service_accounts + } load_service_accounts = [ "serviceAccount:${module.load-project.service_accounts.robots.dataflow}", module.load-sa-df-0.iam_email @@ -35,27 +51,14 @@ locals { module "load-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}" - group_iam = { - (local.groups.data-engineers) = [ - "roles/compute.viewer", - "roles/dataflow.admin", - "roles/dataflow.developer", - "roles/viewer", - ] - } - iam = { - "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] - "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email - ] - "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] - "roles/storage.objectAdmin" = local.load_service_accounts - } + # group_iam = local.group_iam_load + iam = var.project_config.billing_account_id != null ? local.iam_load : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -108,11 +111,11 @@ module "load-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.load-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-lod" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-lod" region = var.region } ] @@ -132,7 +135,7 @@ module "load-nat" { source = "../../../modules/net-cloudnat" count = local.use_shared_vpc ? 0 : 1 project_id = module.load-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-lod" region = var.region router_network = module.load-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index c7f59578..f720fc7f 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -15,31 +15,7 @@ # tfdoc:file:description Orchestration project and VPC. locals { - orch_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_links.orchestration - : values(module.orch-vpc.0.subnet_self_links)[0] - ) - orch_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.orch-vpc.0.self_link - ) - - # Note: This formatting is needed for output purposes since the fabric artifact registry - # module doesn't yet expose the docker usage path of a registry folder in the needed format. - orch_docker_path = format("%s-docker.pkg.dev/%s/%s", - var.region, module.orch-project.project_id, module.orch-artifact-reg.name) -} - -module "orch-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" - group_iam = { + group_iam_orch = { (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", "roles/bigquery.jobUser", @@ -54,7 +30,7 @@ module "orch-project" { "roles/serviceusage.serviceUsageConsumer", ] } - iam = { + iam_orch = { "roles/bigquery.dataEditor" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email, @@ -85,7 +61,34 @@ module "orch-project" { ] "roles/storage.objectViewer" = [module.load-sa-df-0.iam_email] } - oslogin = false + orch_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.orch-vpc.0.subnet_self_links)[0] + ) + orch_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.orch-vpc.0.self_link + ) + + # Note: This formatting is needed for output purposes since the fabric artifact registry + # module doesn't yet expose the docker usage path of a registry folder in the needed format. + orch_docker_path = format("%s-docker.pkg.dev/%s/%s", + var.region, module.orch-project.project_id, module.orch-artifact-reg.name) +} + +module "orch-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" + # group_iam = local.group_iam_orch + iam = var.project_config.billing_account_id != null ? local.iam_orch : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null + oslogin = false services = concat(var.project_services, [ "artifactregistry.googleapis.com", "bigquery.googleapis.com", @@ -133,11 +136,11 @@ module "orch-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.orch-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-orch" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-orch" region = var.region secondary_ip_ranges = { pods = "10.10.8.0/22" @@ -161,7 +164,7 @@ module "orch-nat" { count = local.use_shared_vpc ? 0 : 1 source = "../../../modules/net-cloudnat" project_id = module.orch-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-orch" region = var.region router_network = module.orch-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index c2b9b1e2..63d3f399 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -15,32 +15,13 @@ # tfdoc:file:description Trasformation project and VPC. locals { - transf_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_links.orchestration - : values(module.transf-vpc.0.subnet_self_links)[0] - ) - transf_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.transf-vpc.0.self_link - ) -} - -module "transf-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" - group_iam = { + group_iam_trf = { (local.groups.data-engineers) = [ "roles/bigquery.jobUser", "roles/dataflow.admin", ] } - iam = { + iam_trf = { "roles/bigquery.jobUser" = [ module.transf-sa-bq-0.iam_email, ] @@ -55,6 +36,28 @@ module "transf-project" { "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" ] } + transf_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.transf-vpc.0.subnet_self_links)[0] + ) + transf_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.transf-vpc.0.self_link + ) +} + +module "transf-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" + # group_iam = local.group_iam_trf + iam = var.project_config.billing_account_id != null ? local.iam_orch : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -132,11 +135,11 @@ module "transf-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.transf-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-trf" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-trf" region = var.region } ] @@ -156,7 +159,7 @@ module "transf-nat" { source = "../../../modules/net-cloudnat" count = local.use_shared_vpc ? 0 : 1 project_id = module.transf-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-trf" region = var.region router_network = module.transf-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index 6bc1a28e..d22cf0aa 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -82,14 +82,15 @@ locals { module "dwh-lnd-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_lnd_iam - services = local.dwh_services + # group_iam = local.dwh_group_iam + iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {} + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -98,14 +99,15 @@ module "dwh-lnd-project" { module "dwh-cur-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_iam - services = local.dwh_services + # group_iam = local.dwh_group_iam + iam = var.project_config.billing_account_id != null ? local.dwh_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {} + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -114,14 +116,15 @@ module "dwh-cur-project" { module "dwh-conf-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_iam - services = local.dwh_services + # group_iam = local.dwh_group_iam + iam = var.project_config.billing_account_id != null ? local.dwh_iam : null + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -141,7 +144,7 @@ module "dwh-lnd-bq-0" { module "dwh-cur-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.dwh-cur-project.project_id - id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0" + id = "${replace(var.prefix, "-", "_")}_dwh_cur_bq_0" location = var.location encryption_key = try(local.service_encryption_keys.bq, null) } diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 64df392b..059d6b5e 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -14,14 +14,8 @@ # tfdoc:file:description common project. -module "common-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" - group_iam = { +locals { + group_iam_common = { (local.groups.data-analysts) = [ "roles/datacatalog.viewer", ] @@ -35,7 +29,7 @@ module "common-project" { "roles/datacatalog.admin" ] } - iam = { + iam_common = { "roles/dlp.user" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email @@ -52,6 +46,17 @@ module "common-project" { # local.groups_iam.data-analysts ] } +} +module "common-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" + # group_iam = local.group_iam_common + iam = var.project_config.billing_account_id != null ? local.iam_common : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null services = concat(var.project_services, [ "datacatalog.googleapis.com", "dlp.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf index 4a8071c0..ea8fca09 100644 --- a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf +++ b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf @@ -16,7 +16,7 @@ module "exp-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 08b24b21..f6aaac71 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -233,6 +233,10 @@ To create Cloud Key Management keys in the Data Platform you can uncomment the C To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level. To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules. +### Project Configuration + +The solution can be deployed creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly. + ## Demo pipeline The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features. diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index de033f72..ca5f754f 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -181,6 +181,7 @@ variable "project_config" { description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." type = object({ billing_account_id = optional(string, null) + parent = string project_ids = optional(object({ drop = string load = string @@ -204,7 +205,6 @@ variable "project_config" { } ) }) - default = {} validation { condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null error_message = "At least one attribute should be set." From db6a4f9ac7f2c5bb510eb72b4fd287e21167257f Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 00:45:40 +0100 Subject: [PATCH 03/13] Remove variables --- .../data-solutions/data-platform-foundations/README.md | 6 ++++-- .../data-platform-foundations/variables.tf | 10 ---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index f6aaac71..e1bb5f5e 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -213,9 +213,11 @@ While this blueprint can be used as a standalone deployment, it can also be call ```hcl module "data-platform" { source = "./fabric/blueprints/data-solutions/data-platform-foundations" - billing_account_id = var.billing_account_id - folder_id = var.folder_id organization_domain = "example.com" + project_config = { + billing_account_id = var.billing_account_id + parent = "folders/12345678" + } prefix = "myprefix" } diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index ca5f754f..4ec2fd7e 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -14,11 +14,6 @@ # tfdoc:file:description Terraform Variables. -variable "billing_account_id" { - description = "Billing account id." - type = string -} - variable "composer_config" { description = "Cloud Composer config." type = object({ @@ -119,11 +114,6 @@ variable "data_force_destroy" { default = false } -variable "folder_id" { - description = "Folder to be used for the networking resources in folders/nnnn format." - type = string -} - variable "groups" { description = "User groups." type = map(string) From f4c1fa6c20f28e0f071d81d8c1e6762fc7c64f8d Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 00:56:32 +0100 Subject: [PATCH 04/13] Fix tests. --- .../data-platform-foundations/README.md | 4 ++-- fast/stages/3-data-platform/dev/main.tf | 10 ++++++---- .../data_platform_foundations/fixture/main.tf | 8 +++++--- .../data_platform_foundations/test_plan.py | 2 +- tests/fast/stages/s3_data_platform/common.tfvars | 12 +++++++----- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index e1bb5f5e..d004cf94 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -215,13 +215,13 @@ module "data-platform" { source = "./fabric/blueprints/data-solutions/data-platform-foundations" organization_domain = "example.com" project_config = { - billing_account_id = var.billing_account_id + billing_account_id = "123456-123456-123456" parent = "folders/12345678" } prefix = "myprefix" } -# tftest modules=43 resources=297 +# tftest modules=43 resources=264 ``` ## Customizations diff --git a/fast/stages/3-data-platform/dev/main.tf b/fast/stages/3-data-platform/dev/main.tf index 53d901d1..c600a758 100644 --- a/fast/stages/3-data-platform/dev/main.tf +++ b/fast/stages/3-data-platform/dev/main.tf @@ -18,13 +18,15 @@ module "data-platform" { source = "../../../../blueprints/data-solutions/data-platform-foundations" - billing_account_id = var.billing_account.id composer_config = var.composer_config data_force_destroy = var.data_force_destroy data_catalog_tags = var.data_catalog_tags - folder_id = var.folder_ids.data-platform-dev - groups = var.groups - location = var.location + project_config = { + billing_account_id = var.billing_account.id + parent = var.folder_ids.data-platform-dev + } + groups = var.groups + location = var.location network_config = { host_project = var.host_project_ids.dev-spoke-0 network_self_link = var.vpc_self_links.dev-spoke-0 diff --git a/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf b/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf index 52317d6f..5acb29e8 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf +++ b/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf @@ -17,7 +17,9 @@ module "test" { source = "../../../../../blueprints/data-solutions/data-platform-foundations/" organization_domain = "example.com" - billing_account_id = "123456-123456-123456" - folder_id = "folders/12345678" - prefix = "prefix" + project_config = { + billing_account_id = "123456-123456-123456" + parent = "folders/12345678" + } + prefix = "prefix" } diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py index 785f4705..f3ed2ba0 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py +++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py @@ -23,4 +23,4 @@ def test_resources(e2e_plan_runner): modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 42 - assert len(resources) == 296 + assert len(resources) == 264 diff --git a/tests/fast/stages/s3_data_platform/common.tfvars b/tests/fast/stages/s3_data_platform/common.tfvars index 2ec41d37..97d8bebc 100644 --- a/tests/fast/stages/s3_data_platform/common.tfvars +++ b/tests/fast/stages/s3_data_platform/common.tfvars @@ -1,11 +1,13 @@ automation = { outputs_bucket = "test" } -billing_account = { - id = "012345-67890A-BCDEF0", -} -folder_ids = { - data-platform-dev = "folders/12345678" +project_config = { + billing_account = { + id = "012345-67890A-BCDEF0", + }, + parent = { + data-platform-dev = "folders/12345678" + } } host_project_ids = { dev-spoke-0 = "fast-dev-net-spoke-0" From c523dce3a6890566743a3c9a0c017470d28abd0c Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:00:57 +0100 Subject: [PATCH 05/13] Fix --- blueprints/data-solutions/data-platform-foundations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index d004cf94..98c84ee4 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -221,7 +221,7 @@ module "data-platform" { prefix = "myprefix" } -# tftest modules=43 resources=264 +# tftest modules=43 resources=265 ``` ## Customizations From eda9597f0b01bd58635988f0c23b42761b3c5f54 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:03:38 +0100 Subject: [PATCH 06/13] Fix linting --- .../data-platform-foundations/README.md | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 98c84ee4..027c6299 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -250,20 +250,19 @@ You can find examples in the `[demo](./demo)` folder. | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | -| [folder_id](variables.tf#L122) | Folder to be used for the networking resources in folders/nnnn format. | string | ✓ | | -| [organization_domain](variables.tf#L166) | Organization domain. | string | ✓ | | -| [prefix](variables.tf#L171) | Prefix used for resource names. | string | ✓ | | -| [composer_config](variables.tf#L22) | Cloud Composer config. | object({…}) | | {…} | -| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | -| [data_force_destroy](variables.tf#L116) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [groups](variables.tf#L127) | User groups. | map(string) | | {…} | -| [location](variables.tf#L137) | Location used for multi-regional resources. | string | | "eu" | -| [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_services](variables.tf#L180) | List of core services enabled on all projects. | list(string) | | […] | -| [project_suffix](variables.tf#L191) | Suffix used only for project ids. | string | | null | -| [region](variables.tf#L197) | Region used for regional resources. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L203) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | +| [organization_domain](variables.tf#L156) | Organization domain. | string | ✓ | | +| [prefix](variables.tf#L161) | Prefix used for resource names. | string | ✓ | | +| [project_config](variables.tf#L170) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {…} | +| [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | +| [data_force_destroy](variables.tf#L111) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | +| [groups](variables.tf#L117) | User groups. | map(string) | | {…} | +| [location](variables.tf#L127) | Location used for multi-regional resources. | string | | "eu" | +| [network_config](variables.tf#L133) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_services](variables.tf#L204) | List of core services enabled on all projects. | list(string) | | […] | +| [project_suffix](variables.tf#L215) | Suffix used only for project ids. | string | | null | +| [region](variables.tf#L221) | Region used for regional resources. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L227) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | ## Outputs From 63a81a9b9b1e50f10df9882bfdd052985e9067ef Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:12:19 +0100 Subject: [PATCH 07/13] Fix Fast test --- fast/stages/3-data-platform/dev/variables.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf index 74a5dbe1..d0aad16f 100644 --- a/fast/stages/3-data-platform/dev/variables.tf +++ b/fast/stages/3-data-platform/dev/variables.tf @@ -169,6 +169,14 @@ variable "prefix" { type = string } +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = string + parent = string + }) +} + variable "project_services" { description = "List of core services enabled on all projects." type = list(string) From 970b8ff2557417038db447d0cd4e6a9faa48305b Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:16:22 +0100 Subject: [PATCH 08/13] Fix DP Fast variables. --- fast/stages/3-data-platform/dev/README.md | 11 +++++----- fast/stages/3-data-platform/dev/variables.tf | 21 -------------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index 48d09eaf..518e4516 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -190,6 +190,7 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | [host_project_ids](variables.tf#L120) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | | [organization](variables.tf#L150) | Organization details. | object({…}) | ✓ | | 00-globals | | [prefix](variables.tf#L166) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | +| [project_config](variables.tf#L172) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | | [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | | | [data_catalog_tags](variables.tf#L85) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | | [data_force_destroy](variables.tf#L96) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | @@ -197,11 +198,11 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | [location](variables.tf#L128) | Location used for multi-regional resources. | string | | "eu" | | | [network_config_composer](variables.tf#L134) | Network configurations to use for Composer. | object({…}) | | {…} | | | [outputs_location](variables.tf#L160) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L172) | List of core services enabled on all projects. | list(string) | | […] | | -| [region](variables.tf#L183) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L189) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L201) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L210) | Shared VPC self links. | object({…}) | | null | 2-networking | +| [project_services](variables.tf#L180) | List of core services enabled on all projects. | list(string) | | […] | | +| [region](variables.tf#L191) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L197) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L209) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | +| [vpc_self_links](variables.tf#L218) | Shared VPC self links. | object({…}) | | null | 2-networking | ## Outputs diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf index d0aad16f..392e2dc9 100644 --- a/fast/stages/3-data-platform/dev/variables.tf +++ b/fast/stages/3-data-platform/dev/variables.tf @@ -22,19 +22,6 @@ variable "automation" { }) } -variable "billing_account" { - # tfdoc:variable:source 0-bootstrap - description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false." - type = object({ - id = string - is_org_level = optional(bool, true) - }) - validation { - condition = var.billing_account.is_org_level != null - error_message = "Invalid `null` value for `billing_account.is_org_level`." - } -} - variable "composer_config" { description = "Cloud Composer configuration options." type = object({ @@ -99,14 +86,6 @@ variable "data_force_destroy" { default = false } -variable "folder_ids" { - # tfdoc:variable:source 1-resman - description = "Folder to be used for the networking resources in folders/nnnn format." - type = object({ - data-platform-dev = string - }) -} - variable "groups" { description = "Groups." type = map(string) From 2564c9b06a4b51616e9d3351cfd60628d39e2c78 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:17:08 +0100 Subject: [PATCH 09/13] Fix README --- fast/stages/3-data-platform/dev/README.md | 34 +++++++++++------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index 518e4516..f2990310 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -185,24 +185,22 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | -| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | -| [folder_ids](variables.tf#L102) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman | -| [host_project_ids](variables.tf#L120) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | -| [organization](variables.tf#L150) | Organization details. | object({…}) | ✓ | | 00-globals | -| [prefix](variables.tf#L166) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | -| [project_config](variables.tf#L172) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | -| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | | -| [data_catalog_tags](variables.tf#L85) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | -| [data_force_destroy](variables.tf#L96) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | -| [groups](variables.tf#L110) | Groups. | map(string) | | {…} | | -| [location](variables.tf#L128) | Location used for multi-regional resources. | string | | "eu" | | -| [network_config_composer](variables.tf#L134) | Network configurations to use for Composer. | object({…}) | | {…} | | -| [outputs_location](variables.tf#L160) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L180) | List of core services enabled on all projects. | list(string) | | […] | | -| [region](variables.tf#L191) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L197) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L209) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L218) | Shared VPC self links. | object({…}) | | null | 2-networking | +| [host_project_ids](variables.tf#L99) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | +| [organization](variables.tf#L129) | Organization details. | object({…}) | ✓ | | 00-globals | +| [prefix](variables.tf#L145) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | +| [project_config](variables.tf#L151) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | +| [composer_config](variables.tf#L25) | Cloud Composer configuration options. | object({…}) | | {…} | | +| [data_catalog_tags](variables.tf#L72) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | +| [data_force_destroy](variables.tf#L83) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | +| [groups](variables.tf#L89) | Groups. | map(string) | | {…} | | +| [location](variables.tf#L107) | Location used for multi-regional resources. | string | | "eu" | | +| [network_config_composer](variables.tf#L113) | Network configurations to use for Composer. | object({…}) | | {…} | | +| [outputs_location](variables.tf#L139) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L159) | List of core services enabled on all projects. | list(string) | | […] | | +| [region](variables.tf#L170) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L176) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L188) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | +| [vpc_self_links](variables.tf#L197) | Shared VPC self links. | object({…}) | | null | 2-networking | ## Outputs From 2108b4650deb5814cede7daed89a1548bb1c3117 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 22 Feb 2023 01:36:01 +0100 Subject: [PATCH 10/13] Fix Tests, rely on iam additive. --- .../data-platform-foundations/01-dropoff.tf | 27 ++++---- .../data-platform-foundations/02-load.tf | 29 ++++---- .../03-orchestration.tf | 41 ++++------- .../04-transformation.tf | 19 ++--- .../05-datawarehouse.tf | 69 ++++++++----------- .../data-platform-foundations/06-common.tf | 29 +++----- .../data-platform-foundations/README.md | 10 +-- .../demo/datapipeline_dc_tags.py | 8 +-- .../demo/delete_table.py | 4 +- fast/stages/3-data-platform/dev/variables.tf | 29 +++++--- .../data_platform_foundations/test_plan.py | 2 +- 11 files changed, 120 insertions(+), 147 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index 4c4264d3..46e9a130 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -15,24 +15,22 @@ # tfdoc:file:description drop off project and resources. locals { - group_iam_drp = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/pubsub.editor", - "roles/storage.admin", - ] - } iam_drp = { - "roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email] - "roles/bigquery.user" = [module.load-sa-df-0.iam_email] - "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] + "roles/bigquery.dataEditor" = [ + module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers + ] + "roles/bigquery.user" = [ + module.load-sa-df-0.iam_email, local.groups_iam.data-engineers + ] + "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] "roles/pubsub.subscriber" = [ module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email ] - "roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email] "roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email] "roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email] - "roles/storage.admin" = [module.load-sa-df-0.iam_email] + "roles/storage.objectAdmin" = [ + module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email + ] } } @@ -43,9 +41,8 @@ module "drop-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" - # group_iam = local.group_iam_drp - iam = var.project_config.billing_account_id != null ? local.iam_drp : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null + iam = var.project_config.billing_account_id != null ? local.iam_drp : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index b547f050..9702fce1 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -15,18 +15,15 @@ # tfdoc:file:description Load project and VPC. locals { - group_iam_load = { - (local.groups.data-engineers) = [ - "roles/compute.viewer", - "roles/dataflow.admin", - "roles/dataflow.developer", - "roles/viewer", - ] - } iam_load = { "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email + module.orch-sa-cmp-0.iam_email, + module.load-sa-df-0.iam_email, + local.groups_iam.data-engineers + ] + "roles/dataflow.developer" = [ + local.groups_iam.data-engineers ] "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] "roles/storage.objectAdmin" = local.load_service_accounts @@ -56,9 +53,8 @@ module "load-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}" - # group_iam = local.group_iam_load - iam = var.project_config.billing_account_id != null ? local.iam_load : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null + iam = var.project_config.billing_account_id != null ? local.iam_load : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -90,8 +86,13 @@ module "load-sa-df-0" { name = "load-df-0" display_name = "Data platform Dataflow load service account" iam = { - "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] - "roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email] + "roles/iam.serviceAccountTokenCreator" = [ + local.groups_iam.data-engineers, + module.orch-sa-cmp-0.iam_email + ], + "roles/iam.serviceAccountUser" = [ + module.orch-sa-cmp-0.iam_email + ] } } diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index f720fc7f..fc0eda12 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -15,29 +15,22 @@ # tfdoc:file:description Orchestration project and VPC. locals { - group_iam_orch = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/bigquery.jobUser", - "roles/cloudbuild.builds.editor", - "roles/composer.admin", - "roles/composer.environmentAndStorageObjectAdmin", - "roles/iap.httpsResourceAccessor", - "roles/iam.serviceAccountUser", - "roles/storage.objectAdmin", - "roles/storage.admin", - "roles/artifactregistry.admin", - "roles/serviceusage.serviceUsageConsumer", - ] - } iam_orch = { + "roles/artifactregistry.admin" = [local.groups_iam.data-engineers] + "roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email] "roles/bigquery.dataEditor" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email, + local.groups_iam.data-engineers ] "roles/bigquery.jobUser" = [ module.orch-sa-cmp-0.iam_email, + local.groups_iam.data-engineers ] + "roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers] + "roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email] + "roles/composer.admin" = [local.groups_iam.data-engineers] + "roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers] "roles/composer.ServiceAgentV2Ext" = [ "serviceAccount:${module.orch-project.service_accounts.robots.composer}" ] @@ -45,19 +38,16 @@ locals { module.orch-sa-cmp-0.iam_email ] "roles/iam.serviceAccountUser" = [ - module.orch-sa-cmp-0.iam_email + module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers ] + "roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers] + "roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers] "roles/storage.objectAdmin" = [ module.orch-sa-cmp-0.iam_email, module.orch-sa-df-build.iam_email, "serviceAccount:${module.orch-project.service_accounts.robots.composer}", "serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}", - ] - "roles/artifactregistry.reader" = [ - module.load-sa-df-0.iam_email, - ] - "roles/cloudbuild.serviceAgent" = [ - module.orch-sa-df-build.iam_email, + local.groups_iam.data-engineers ] "roles/storage.objectViewer" = [module.load-sa-df-0.iam_email] } @@ -85,10 +75,9 @@ module "orch-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" - # group_iam = local.group_iam_orch - iam = var.project_config.billing_account_id != null ? local.iam_orch : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null - oslogin = false + iam = var.project_config.billing_account_id != null ? local.iam_orch : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null + oslogin = false services = concat(var.project_services, [ "artifactregistry.googleapis.com", "bigquery.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index 63d3f399..394adedf 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -15,22 +15,14 @@ # tfdoc:file:description Trasformation project and VPC. locals { - group_iam_trf = { - (local.groups.data-engineers) = [ - "roles/bigquery.jobUser", - "roles/dataflow.admin", - ] - } iam_trf = { "roles/bigquery.jobUser" = [ - module.transf-sa-bq-0.iam_email, + module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers ] "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, - ] - "roles/dataflow.worker" = [ - module.transf-sa-df-0.iam_email + module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers ] + "roles/dataflow.worker" = [module.transf-sa-df-0.iam_email] "roles/storage.objectAdmin" = [ module.transf-sa-df-0.iam_email, "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" @@ -55,9 +47,8 @@ module "transf-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" - # group_iam = local.group_iam_trf - iam = var.project_config.billing_account_id != null ? local.iam_orch : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null + iam = var.project_config.billing_account_id != null ? local.iam_trf : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index d22cf0aa..67c43dae 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -15,54 +15,48 @@ # tfdoc:file:description Data Warehouse projects. locals { - dwh_group_iam = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/storage.admin", - ], - (local.groups.data-analysts) = [ - "roles/bigquery.dataViewer", - "roles/bigquery.jobUser", - "roles/bigquery.metadataViewer", - "roles/bigquery.user", - "roles/datacatalog.viewer", - "roles/datacatalog.tagTemplateViewer", - "roles/storage.objectViewer", - ] - } dwh_lnd_iam = { "roles/bigquery.dataOwner" = [ module.load-sa-df-0.iam_email, + ] + "roles/bigquery.dataViewer" = [ module.transf-sa-df-0.iam_email, module.transf-sa-bq-0.iam_email, + local.groups_iam.data-engineers ] "roles/bigquery.jobUser" = [ - module.load-sa-df-0.iam_email, - ] - "roles/datacatalog.categoryAdmin" = [ - module.transf-sa-bq-0.iam_email - ] - "roles/storage.objectCreator" = [ - module.load-sa-df-0.iam_email, + module.load-sa-df-0.iam_email, local.groups_iam.data-engineers ] + "roles/datacatalog.categoryAdmin" = [module.transf-sa-bq-0.iam_email] + "roles/datacatalog.tagTemplateViewer" = [local.groups_iam.data-engineers] + "roles/datacatalog.viewer" = [local.groups_iam.data-engineers] + "roles/storage.objectCreator" = [module.load-sa-df-0.iam_email] + "roles/storage.objectViewer" = [local.groups_iam.data-engineers] } dwh_iam = { "roles/bigquery.dataOwner" = [ module.transf-sa-df-0.iam_email, module.transf-sa-bq-0.iam_email, ] + "roles/bigquery.dataViewer" = [ + local.groups_iam.data-analysts, + local.groups_iam.data-engineers + ] "roles/bigquery.jobUser" = [ module.transf-sa-bq-0.iam_email, + local.groups_iam.data-analysts, + local.groups_iam.data-engineers ] - "roles/datacatalog.categoryAdmin" = [ - module.load-sa-df-0.iam_email + "roles/datacatalog.tagTemplateViewer" = [ + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] - "roles/storage.objectCreator" = [ - module.transf-sa-df-0.iam_email, + "roles/datacatalog.viewer" = [ + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] "roles/storage.objectViewer" = [ - module.transf-sa-df-0.iam_email, + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] + "roles/storage.objectAdmin" = [module.transf-sa-df-0.iam_email] } dwh_services = concat(var.project_services, [ "bigquery.googleapis.com", @@ -87,10 +81,9 @@ module "dwh-lnd-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" - # group_iam = local.dwh_group_iam - iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {} - iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {} - services = local.dwh_services + iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {} + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -104,10 +97,9 @@ module "dwh-cur-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" - # group_iam = local.dwh_group_iam - iam = var.project_config.billing_account_id != null ? local.dwh_iam : {} - iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {} - services = local.dwh_services + iam = var.project_config.billing_account_id != null ? local.dwh_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {} + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -121,10 +113,9 @@ module "dwh-conf-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" - # group_iam = local.dwh_group_iam - iam = var.project_config.billing_account_id != null ? local.dwh_iam : null - iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null - services = local.dwh_services + iam = var.project_config.billing_account_id != null ? local.dwh_iam : null + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 059d6b5e..5a84ee77 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -15,29 +15,21 @@ # tfdoc:file:description common project. locals { - group_iam_common = { - (local.groups.data-analysts) = [ - "roles/datacatalog.viewer", - ] - (local.groups.data-engineers) = [ - "roles/dlp.reader", - "roles/dlp.user", - "roles/dlp.estimatesAdmin", - ] - (local.groups.data-security) = [ - "roles/dlp.admin", - "roles/datacatalog.admin" - ] - } iam_common = { + "roles/dlp.admin" = [local.groups_iam.data-security] + "roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers] + "roles/dlp.reader" = [local.groups_iam.data-engineers] "roles/dlp.user" = [ module.load-sa-df-0.iam_email, - module.transf-sa-df-0.iam_email + module.transf-sa-df-0.iam_email, + local.groups_iam.data-engineers ] + "roles/datacatalog.admin" = [local.groups_iam.data-security] "roles/datacatalog.viewer" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email, - module.transf-sa-bq-0.iam_email + module.transf-sa-bq-0.iam_email, + local.groups_iam.data-analysts ] "roles/datacatalog.categoryFineGrainedReader" = [ module.transf-sa-df-0.iam_email, @@ -54,9 +46,8 @@ module "common-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" - # group_iam = local.group_iam_common - iam = var.project_config.billing_account_id != null ? local.iam_common : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null + iam = var.project_config.billing_account_id != null ? local.iam_common : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null services = concat(var.project_services, [ "datacatalog.googleapis.com", "dlp.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 027c6299..d48ae6cc 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -215,13 +215,13 @@ module "data-platform" { source = "./fabric/blueprints/data-solutions/data-platform-foundations" organization_domain = "example.com" project_config = { - billing_account_id = "123456-123456-123456" - parent = "folders/12345678" - } - prefix = "myprefix" + billing_account_id = "123456-123456-123456" + parent = "folders/12345678" + } + prefix = "myprefix" } -# tftest modules=43 resources=265 +# tftest modules=43 resources=278 ``` ## Customizations diff --git a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py index 4b15eaab..86b8e5bb 100644 --- a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py +++ b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py @@ -123,7 +123,7 @@ with models.DAG( task_id="upsert_table_customers", project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, - impersonation_chain=[TRF_SA_DF], + impersonation_chain=[LOD_SA_DF], table_resource={ "tableReference": {"tableId": "customers"}, }, @@ -133,7 +133,7 @@ with models.DAG( task_id="upsert_table_purchases", project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], table_resource={ "tableReference": {"tableId": "purchases"} }, @@ -167,7 +167,7 @@ with models.DAG( project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, table_id="customers", - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], include_policy_tags=True, schema_fields_updates=[ { "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" }, @@ -182,7 +182,7 @@ with models.DAG( project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, table_id="purchases", - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], include_policy_tags=True, schema_fields_updates=[ { "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" }, diff --git a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py index dc0c954b..bade0388 100644 --- a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py +++ b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py @@ -122,13 +122,13 @@ with models.DAG( delete_table_customers = BigQueryDeleteTableOperator( task_id="delete_table_customers", deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers", - impersonation_chain=[TRF_SA_DF] + impersonation_chain=[LOD_SA_DF] ) delete_table_purchases = BigQueryDeleteTableOperator( task_id="delete_table_purchases", deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases", - impersonation_chain=[TRF_SA_DF] + impersonation_chain=[LOD_SA_DF] ) delete_table_customer_purchase_curated = BigQueryDeleteTableOperator( diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf index 392e2dc9..74a5dbe1 100644 --- a/fast/stages/3-data-platform/dev/variables.tf +++ b/fast/stages/3-data-platform/dev/variables.tf @@ -22,6 +22,19 @@ variable "automation" { }) } +variable "billing_account" { + # tfdoc:variable:source 0-bootstrap + description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false." + type = object({ + id = string + is_org_level = optional(bool, true) + }) + validation { + condition = var.billing_account.is_org_level != null + error_message = "Invalid `null` value for `billing_account.is_org_level`." + } +} + variable "composer_config" { description = "Cloud Composer configuration options." type = object({ @@ -86,6 +99,14 @@ variable "data_force_destroy" { default = false } +variable "folder_ids" { + # tfdoc:variable:source 1-resman + description = "Folder to be used for the networking resources in folders/nnnn format." + type = object({ + data-platform-dev = string + }) +} + variable "groups" { description = "Groups." type = map(string) @@ -148,14 +169,6 @@ variable "prefix" { type = string } -variable "project_config" { - description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." - type = object({ - billing_account_id = string - parent = string - }) -} - variable "project_services" { description = "List of core services enabled on all projects." type = list(string) diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py index f3ed2ba0..630944f2 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py +++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py @@ -23,4 +23,4 @@ def test_resources(e2e_plan_runner): modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 42 - assert len(resources) == 264 + assert len(resources) == 277 From ac75cbe71ac2ae57162319d16cc557e468581126 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 22 Feb 2023 01:38:44 +0100 Subject: [PATCH 11/13] Fix lint. --- fast/stages/3-data-platform/dev/README.md | 33 ++++++++++++----------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index f2990310..48d09eaf 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -185,22 +185,23 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | -| [host_project_ids](variables.tf#L99) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | -| [organization](variables.tf#L129) | Organization details. | object({…}) | ✓ | | 00-globals | -| [prefix](variables.tf#L145) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | -| [project_config](variables.tf#L151) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | -| [composer_config](variables.tf#L25) | Cloud Composer configuration options. | object({…}) | | {…} | | -| [data_catalog_tags](variables.tf#L72) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | -| [data_force_destroy](variables.tf#L83) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | -| [groups](variables.tf#L89) | Groups. | map(string) | | {…} | | -| [location](variables.tf#L107) | Location used for multi-regional resources. | string | | "eu" | | -| [network_config_composer](variables.tf#L113) | Network configurations to use for Composer. | object({…}) | | {…} | | -| [outputs_location](variables.tf#L139) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L159) | List of core services enabled on all projects. | list(string) | | […] | | -| [region](variables.tf#L170) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L176) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L188) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L197) | Shared VPC self links. | object({…}) | | null | 2-networking | +| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | +| [folder_ids](variables.tf#L102) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman | +| [host_project_ids](variables.tf#L120) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | +| [organization](variables.tf#L150) | Organization details. | object({…}) | ✓ | | 00-globals | +| [prefix](variables.tf#L166) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | +| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | | +| [data_catalog_tags](variables.tf#L85) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | +| [data_force_destroy](variables.tf#L96) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | +| [groups](variables.tf#L110) | Groups. | map(string) | | {…} | | +| [location](variables.tf#L128) | Location used for multi-regional resources. | string | | "eu" | | +| [network_config_composer](variables.tf#L134) | Network configurations to use for Composer. | object({…}) | | {…} | | +| [outputs_location](variables.tf#L160) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L172) | List of core services enabled on all projects. | list(string) | | […] | | +| [region](variables.tf#L183) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L189) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L201) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | +| [vpc_self_links](variables.tf#L210) | Shared VPC self links. | object({…}) | | null | 2-networking | ## Outputs From e39be7b01d18f19e509cd82acf5fd5c026a031b2 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 22 Feb 2023 13:02:29 +0100 Subject: [PATCH 12/13] Fix --- tests/fast/stages/s3_data_platform/common.tfvars | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/fast/stages/s3_data_platform/common.tfvars b/tests/fast/stages/s3_data_platform/common.tfvars index 97d8bebc..2ec41d37 100644 --- a/tests/fast/stages/s3_data_platform/common.tfvars +++ b/tests/fast/stages/s3_data_platform/common.tfvars @@ -1,13 +1,11 @@ automation = { outputs_bucket = "test" } -project_config = { - billing_account = { - id = "012345-67890A-BCDEF0", - }, - parent = { - data-platform-dev = "folders/12345678" - } +billing_account = { + id = "012345-67890A-BCDEF0", +} +folder_ids = { + data-platform-dev = "folders/12345678" } host_project_ids = { dev-spoke-0 = "fast-dev-net-spoke-0" From b279c083a0bfdd3abb2b90b717609d32ece9caca Mon Sep 17 00:00:00 2001 From: lcaggio Date: Thu, 23 Feb 2023 11:54:16 +0100 Subject: [PATCH 13/13] Fix README and IAM files --- .../data-platform-foundations/IAM.md | 29 +++++++++---------- .../data-platform-foundations/README.md | 6 +++- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/IAM.md b/blueprints/data-solutions/data-platform-foundations/IAM.md index dd898bd7..b982f8c4 100644 --- a/blueprints/data-solutions/data-platform-foundations/IAM.md +++ b/blueprints/data-solutions/data-platform-foundations/IAM.md @@ -17,51 +17,48 @@ Legend: + additive, • conditional. | members | roles | |---|---| -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) | |drp-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) | |drp-cs-0
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | |drp-ps-0
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) | -|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |orc-cmp-0
serviceAccount|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | ## Project dwh-conf | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| -|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | |trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | ## Project dwh-cur | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| -|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | |trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | ## Project dwh-lnd | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| |load-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | -|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) | +|trf-bq-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | +|trf-df-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) | ## Project lod | members | roles | |---|---| -|gcp-data-engineers
group|[roles/compute.viewer](https://cloud.google.com/iam/docs/understanding-roles#compute.viewer)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer)
[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) | +|gcp-data-engineers
group|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) | |SERVICE_IDENTITY_dataflow-service-producer-prod
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| |load-df-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | @@ -71,7 +68,7 @@ Legend: + additive, • conditional. | members | roles | |---|---| -|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_cloudcomposer-accounts
serviceAccount|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_gcp-sa-cloudbuild
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index d48ae6cc..ad087216 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -237,7 +237,11 @@ To do this, you need to remove IAM binging at project-level for the `data-analys ### Project Configuration -The solution can be deployed creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly. +The solution can be deployed by creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly. + +When you rely on existing projects, the blueprint is designed to rely on different projects configuring IAM binding with an additive approach. For discovery or experimentation purposes, you may also configure `project_config.project_ids` to point different projects to one project with the granularity you need. For example, deploy resources from the 'load' project with resources in the 'transformation' project. + +Once you have identified the required project granularity for your use case, we suggest adapting the terraform script accordingly and relying on authoritative IAM binding. ## Demo pipeline