From 2412f4d1ad97e3909bc6e9a0c92c252273683acc Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Sat, 12 Feb 2022 00:50:38 +0100 Subject: [PATCH] Switch to multiregion GCS and BQ. Add IAM.MD --- .../data-platform-foundations/01-landing.tf | 6 +- .../data-platform-foundations/02-load.tf | 4 +- .../03-orchestration.tf | 4 +- .../04-transformation.tf | 4 +- .../data-platform-foundations/05-datalake.tf | 24 ++--- .../data-platform-foundations/06-common.tf | 10 +- .../data-platform-foundations/IAM.md | 96 +++++++++++++++++++ .../data-platform-foundations/README.md | 28 +++--- .../data-platform-foundations/variables.tf | 6 ++ 9 files changed, 142 insertions(+), 40 deletions(-) create mode 100644 examples/data-solutions/data-platform-foundations/IAM.md diff --git a/examples/data-solutions/data-platform-foundations/01-landing.tf b/examples/data-solutions/data-platform-foundations/01-landing.tf index 460fc283..b1e08375 100644 --- a/examples/data-solutions/data-platform-foundations/01-landing.tf +++ b/examples/data-solutions/data-platform-foundations/01-landing.tf @@ -83,8 +83,8 @@ module "land-cs-0" { project_id = module.land-project.project_id prefix = var.prefix name = "lnd-cs-0" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) force_destroy = var.data_force_destroy # retention_policy = { @@ -134,6 +134,6 @@ module "land-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.land-project.project_id id = "${replace(var.prefix, "-", "_")}lnd_bq_0" - location = var.region + location = var.location encryption_key = try(local.service_encryption_keys.bq, null) } diff --git a/examples/data-solutions/data-platform-foundations/02-load.tf b/examples/data-solutions/data-platform-foundations/02-load.tf index 6696f4d3..5ffb0e2b 100644 --- a/examples/data-solutions/data-platform-foundations/02-load.tf +++ b/examples/data-solutions/data-platform-foundations/02-load.tf @@ -108,8 +108,8 @@ module "load-cs-df-0" { project_id = module.load-project.project_id prefix = var.prefix name = "load-cs-0" - storage_class = "REGIONAL" - location = var.region + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) } diff --git a/examples/data-solutions/data-platform-foundations/03-orchestration.tf b/examples/data-solutions/data-platform-foundations/03-orchestration.tf index a2c53940..70576531 100644 --- a/examples/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/examples/data-solutions/data-platform-foundations/03-orchestration.tf @@ -125,8 +125,8 @@ module "orch-cs-0" { project_id = module.orch-project.project_id prefix = var.prefix name = "orc-cs-0" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) } diff --git a/examples/data-solutions/data-platform-foundations/04-transformation.tf b/examples/data-solutions/data-platform-foundations/04-transformation.tf index 2cc7f420..62cacf3a 100644 --- a/examples/data-solutions/data-platform-foundations/04-transformation.tf +++ b/examples/data-solutions/data-platform-foundations/04-transformation.tf @@ -107,8 +107,8 @@ module "transf-cs-df-0" { project_id = module.transf-project.project_id prefix = var.prefix name = "trf-cs-0" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) } diff --git a/examples/data-solutions/data-platform-foundations/05-datalake.tf b/examples/data-solutions/data-platform-foundations/05-datalake.tf index 71ad0447..8bf7c501 100644 --- a/examples/data-solutions/data-platform-foundations/05-datalake.tf +++ b/examples/data-solutions/data-platform-foundations/05-datalake.tf @@ -138,7 +138,7 @@ module "lake-0-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.lake-0-project.project_id id = "${replace(var.prefix, "-", "_")}_dtl_0_bq_0" - location = var.region + location = var.location encryption_key = try(local.service_encryption_keys.bq, null) } @@ -146,7 +146,7 @@ module "lake-1-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.lake-1-project.project_id id = "${replace(var.prefix, "-", "_")}_dtl_1_bq_0" - location = var.region + location = var.location encryption_key = try(local.service_encryption_keys.bq, null) } @@ -154,7 +154,7 @@ module "lake-2-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.lake-2-project.project_id id = "${replace(var.prefix, "-", "_")}_dtl_2_bq_0" - location = var.region + location = var.location encryption_key = try(local.service_encryption_keys.bq, null) } @@ -162,7 +162,7 @@ module "lake-plg-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.lake-plg-project.project_id id = "${replace(var.prefix, "-", "_")}_dtl_plg_bq_0" - location = var.region + location = var.location encryption_key = try(local.service_encryption_keys.bq, null) } @@ -173,8 +173,8 @@ module "lake-0-cs-0" { project_id = module.lake-0-project.project_id prefix = var.prefix name = "dtl-0-cs-0" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) force_destroy = var.data_force_destroy } @@ -184,8 +184,8 @@ module "lake-1-cs-0" { project_id = module.lake-1-project.project_id prefix = var.prefix name = "dtl-1-cs-0" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) force_destroy = var.data_force_destroy } @@ -195,8 +195,8 @@ module "lake-2-cs-0" { project_id = module.lake-2-project.project_id prefix = var.prefix name = "dtl-2-cs-0" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) force_destroy = var.data_force_destroy } @@ -206,8 +206,8 @@ module "lake-plg-cs-0" { project_id = module.lake-plg-project.project_id prefix = var.prefix name = "dtl-plg-cs-0" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) force_destroy = var.data_force_destroy } diff --git a/examples/data-solutions/data-platform-foundations/06-common.tf b/examples/data-solutions/data-platform-foundations/06-common.tf index 126f947e..59279d23 100644 --- a/examples/data-solutions/data-platform-foundations/06-common.tf +++ b/examples/data-solutions/data-platform-foundations/06-common.tf @@ -46,10 +46,10 @@ module "common-project" { # module "cmn-kms-0" { # source = "../../../modules/kms" -# project_id = module.cmn-prj.project_id +# project_id = module.common-project.project_id # keyring = { # name = "${var.prefix}-kr-global", -# location = var.location_config.region +# location = "global" # } # keys = { # pubsub = null @@ -58,10 +58,10 @@ module "common-project" { # module "cmn-kms-1" { # source = "../../../modules/kms" -# project_id = module.cmn-prj.project_id +# project_id = module.common-project.project_id # keyring = { # name = "${var.prefix}-kr-mregional", -# location = var.location_config.region +# location = var.location # } # keys = { # bq = null @@ -74,7 +74,7 @@ module "common-project" { # project_id = module.cmn-prj.project_id # keyring = { # name = "${var.prefix}-kr-regional", -# location = var.location_config.region +# location = var.region # } # keys = { # composer = null diff --git a/examples/data-solutions/data-platform-foundations/IAM.md b/examples/data-solutions/data-platform-foundations/IAM.md new file mode 100644 index 00000000..07db8e0e --- /dev/null +++ b/examples/data-solutions/data-platform-foundations/IAM.md @@ -0,0 +1,96 @@ +# IAM bindings reference + +Legend: + additive, conditional. + +## Project cmn + +| members | roles | +|---|---| +|gcp-data-engineers
group|[roles/dlp.estimatesAdmin](https://cloud.google.com/iam/docs/understanding-roles#dlp.estimatesAdmin)
[roles/dlp.reader](https://cloud.google.com/iam/docs/understanding-roles#dlp.reader)
[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) | +|gcp-data-security
group|[roles/dlp.admin](https://cloud.google.com/iam/docs/understanding-roles#dlp.admin) | +|load-df-0
serviceAccount|[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) | +|trf-df-0
serviceAccount|[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) | + +## Project dtl-0 + +| members | roles | +|---|---| +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|load-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | +|orc-cmp-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | + +## Project dtl-1 + +| members | roles | +|---|---| +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|load-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | +|orc-cmp-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | + +## Project dtl-2 + +| members | roles | +|---|---| +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|load-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | +|orc-cmp-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | + +## Project dtl-plg + +| members | roles | +|---|---| +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|load-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | +|orc-cmp-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | + +## Project lnd + +| members | roles | +|---|---| +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|lnd-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) | +|lnd-cs-0
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | +|lnd-ps-0
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) | +|load-df-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|orc-cmp-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | + +## Project lod + +| members | roles | +|---|---| +|gcp-data-engineers
group|[roles/compute.viewer](https://cloud.google.com/iam/docs/understanding-roles#compute.viewer)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer)
[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) | +|load-df-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|orc-cmp-0
serviceAccount|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) | +|service-256115644733
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | + +## Project orc + +| members | roles | +|---|---| +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/compute.networkUser](https://cloud.google.com/iam/docs/understanding-roles#compute.networkUser)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|load-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|orc-cmp-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|service-396869095309
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|trf-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | + +## Project trf + +| members | roles | +|---|---| +|gcp-data-engineers
group|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) | +|orc-cmp-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|service-849847706799
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|trf-bq-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | +|trf-df-0
serviceAccount|[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | diff --git a/examples/data-solutions/data-platform-foundations/README.md b/examples/data-solutions/data-platform-foundations/README.md index 68a24134..a14d5e26 100644 --- a/examples/data-solutions/data-platform-foundations/README.md +++ b/examples/data-solutions/data-platform-foundations/README.md @@ -65,6 +65,8 @@ Service account creation follows the least privilege principle, performing a sin |transformation-sa|-|READ/WRITE|READ/WRITE|READ/WRITE| |orchestration-sa|-|-|-|-| +A full reference of IAM roles managed by the DP [is available here](./IAM.md). + Using of service account keys within a data pipeline exposes to several security risks deriving from a credentials leak. This example shows how to leverage impersonation to avoid the need of creating keys. ### User groups @@ -132,6 +134,7 @@ service_encryption_keys = { dataflow = "KEY_URL_REGIONAL" storage = "KEY_URL_MULTIREGIONAL" pubsub = "KEY_URL_MULTIREGIONAL" +} ``` This step is optional and depends on customer policies and security best practices. @@ -181,14 +184,10 @@ The DP is meant to be executed by a Service Account (or a regular user) having t There are three sets of variables you will need to fill in: ```hcl -prefix = "myco" -project_create = { - parent = "folders/123456789012" - billing_account_id = "111111-222222-333333" -} -organization = { - domain = "domain.com" -} +billing_account_id = "111111-222222-333333" +older_id = "folders/123456789012" +organization_domain = "domain.com" +prefix = "myco" ``` For more fine details check variables on [`variables.tf`](./variables.tf) and update according to the desired configuration. Remember to create team groups described [below](#groups). @@ -232,14 +231,15 @@ Description of commands: |---|---|:---:|:---:|:---:| | [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | | [folder_id](variables.tf#L41) | Folder to be used for the networking resources in folders/nnnn format. | string | ✓ | | -| [organization_domain](variables.tf#L79) | Organization domain. | string | ✓ | | -| [prefix](variables.tf#L84) | Unique prefix used for resource names. | string | ✓ | | +| [organization_domain](variables.tf#L85) | Organization domain. | string | ✓ | | +| [prefix](variables.tf#L90) | Unique prefix used for resource names. | string | ✓ | | | [composer_config](variables.tf#L22) | | object({…}) | | {…} | | [data_force_destroy](variables.tf#L35) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [groups](variables.tf#L46) | Groups. | map(string) | | {…} | -| [network_config](variables.tf#L56) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_services](variables.tf#L89) | List of core services enabled on all projects. | list(string) | | […] | -| [region](variables.tf#L100) | Region used for regional resources. | string | | "europe-west1" | +| [groups](variables.tf#L52) | Groups. | map(string) | | {…} | +| [location](variables.tf#L46) | Location used for multi-regional resources. | string | | "eu" | +| [network_config](variables.tf#L62) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_services](variables.tf#L95) | List of core services enabled on all projects. | list(string) | | […] | +| [region](variables.tf#L106) | Region used for regional resources. | string | | "europe-west1" | ## Outputs diff --git a/examples/data-solutions/data-platform-foundations/variables.tf b/examples/data-solutions/data-platform-foundations/variables.tf index 35dc3862..e947fce1 100644 --- a/examples/data-solutions/data-platform-foundations/variables.tf +++ b/examples/data-solutions/data-platform-foundations/variables.tf @@ -43,6 +43,12 @@ variable "folder_id" { type = string } +variable "location" { + description = "Location used for multi-regional resources." + type = string + default = "eu" +} + variable "groups" { description = "Groups." type = map(string)