Fix KMS

examples/data-solutions/dp-foundation/01-lnd-datastorage.tf

@@ -42,7 +42,7 @@ module "lnd-cs-0" {
   location      = var.region
   storage_class = "REGIONAL"
   # retention_policy = local.lnd_bucket_retention_policy
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.storage, null) : null
   force_destroy  = var.data_force_destroy
 }
 
@@ -89,5 +89,5 @@ module "lnd-bq-0" {
   project_id     = module.lnd-prj.project_id
   id             = "${replace(local.prefix_lnd, "-", "_")}_bq_0"
   location       = var.region
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.bq, null) : null
 }

examples/data-solutions/dp-foundation/01-lnd-main.tf.tf

@@ -79,7 +79,7 @@ module "lnd-prj" {
   # additive IAM bindings avoid disrupting bindings in existing project
   iam          = var.project_create != null ? local.iam_lnd : {}
   iam_additive = var.project_create == null ? local.iam_lnd : {}
-  # group_iam    = local.group_iam_lnd
+  group_iam    = local.group_iam_lnd
   services = concat(var.project_services, [
     "bigquery.googleapis.com",
     "bigqueryreservation.googleapis.com",
@@ -89,4 +89,9 @@ module "lnd-prj" {
     "storage.googleapis.com",
     "storage-component.googleapis.com",
   ])
+  service_encryption_key_ids = {
+    bq      = [try(var.service_encryption_keys.bq, null)]
+    pubsub  = [try(var.service_encryption_keys.pubsub, null)]
+    storage = [try(var.service_encryption_keys.storage, null)]
+  }
 }

examples/data-solutions/dp-foundation/02-lod-datastorage.tf

@@ -38,5 +38,5 @@ module "lod-cs-df-0" {
   prefix         = local.prefix_lod
   storage_class  = "REGIONAL"
   location       = var.region
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.storage, null) : null
 }

examples/data-solutions/dp-foundation/02-lod-main.tf

@@ -73,4 +73,9 @@ module "lod-prj" {
     "storage.googleapis.com",
     "storage-component.googleapis.com"
   ])
+  service_encryption_key_ids = {
+    pubsub   = [try(var.service_encryption_keys.pubsub, null)]
+    dataflow = [try(var.service_encryption_keys.dataflow, null)]
+    storage  = [try(var.service_encryption_keys.storage, null)]
+  }
 }

examples/data-solutions/dp-foundation/03-orc-composer.tf

@@ -89,9 +89,9 @@ resource "google_composer_environment" "orc-cmp-0" {
     }
 
     dynamic "encryption_config" {
-      for_each = can(module.kms[0].keys.key-cmp.id) ? { 1 = 1 } : {}
+      for_each = var.service_encryption_keys != null ? { 1 = 1 } : {}
       content {
-        kms_key_name = var.cmek_encryption ? try(module.kms[0].keys.key-cmp.id, null) : null
+        kms_key_name = var.service_encryption_keys != null ? try(var.service_encryption_keys.composer, null) : null
       }
     }
 

examples/data-solutions/dp-foundation/03-orc-datastorage.tf

@@ -23,5 +23,5 @@ module "orc-cs-0" {
   prefix         = local.prefix_orc
   location       = var.region
   storage_class  = "REGIONAL"
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.storage, null) : null
 }

examples/data-solutions/dp-foundation/03-orc-main.tf

@@ -93,4 +93,8 @@ module "orc-prj" {
     "storage.googleapis.com",
     "storage-component.googleapis.com"
   ])
+  service_encryption_key_ids = {
+    composer = [try(var.service_encryption_keys.composer, null)]
+    storage  = [try(var.service_encryption_keys.storage, null)]
+  }
 }

examples/data-solutions/dp-foundation/04-trf-datastorage.tf

@@ -39,7 +39,7 @@ module "trf-cs-df-0" {
   prefix         = local.prefix_trf
   location       = var.region
   storage_class  = "REGIONAL"
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.storage, null) : null
 }
 
 ###############################################################################

examples/data-solutions/dp-foundation/04-trf-main.tf

@@ -69,4 +69,8 @@ module "trf-prj" {
     "storage.googleapis.com",
     "storage-component.googleapis.com"
   ])
+  service_encryption_key_ids = {
+    dataflow = [try(var.service_encryption_keys.dataflow, null)]
+    storage  = [try(var.service_encryption_keys.storage, null)]
+  }
 }

examples/data-solutions/dp-foundation/05-dtl-datastorage.tf

@@ -21,7 +21,7 @@ module "dtl-0-bq-0" {
   project_id     = module.dtl-0-prj.project_id
   id             = "${replace(local.prefix_dtl, "-", "_")}_0_bq_0"
   location       = var.region
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.bq, null) : null
 }
 
 module "dtl-1-bq-0" {
@@ -29,7 +29,7 @@ module "dtl-1-bq-0" {
   project_id     = module.dtl-1-prj.project_id
   id             = "${replace(local.prefix_dtl, "-", "_")}_1_bq_0"
   location       = var.region
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.bq, null) : null
 }
 
 module "dtl-2-bq-0" {
@@ -37,7 +37,7 @@ module "dtl-2-bq-0" {
   project_id     = module.dtl-2-prj.project_id
   id             = "${replace(local.prefix_dtl, "-", "_")}_2_bq_0"
   location       = var.region
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.bq, null) : null
 }
 
 module "dtl-exp-bq-0" {
@@ -45,7 +45,7 @@ module "dtl-exp-bq-0" {
   project_id     = module.dtl-exp-prj.project_id
   id             = "${replace(local.prefix_dtl, "-", "_")}_exp_bq_0"
   location       = var.region
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-bq.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.bq, null) : null
 }
 
 ###############################################################################
@@ -59,7 +59,7 @@ module "dtl-0-cs-0" {
   prefix         = local.prefix_dtl
   location       = var.region
   storage_class  = "REGIONAL"
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.storage, null) : null
   force_destroy  = var.data_force_destroy
 }
 
@@ -70,7 +70,7 @@ module "dtl-1-cs-0" {
   prefix         = local.prefix_dtl
   location       = var.region
   storage_class  = "REGIONAL"
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.storage, null) : null
   force_destroy  = var.data_force_destroy
 }
 
@@ -81,7 +81,7 @@ module "dtl-2-cs-0" {
   prefix         = local.prefix_dtl
   location       = var.region
   storage_class  = "REGIONAL"
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.storage, null) : null
   force_destroy  = var.data_force_destroy
 }
 
@@ -92,6 +92,6 @@ module "dtl-exp-cs-0" {
   prefix         = local.prefix_dtl
   location       = var.region
   storage_class  = "REGIONAL"
-  encryption_key = var.cmek_encryption ? try(module.kms[0].keys.key-gcs.id, null) : null
+  encryption_key = var.service_encryption_keys != null ? try(var.service_encryption_keys.storage, null) : null
   force_destroy  = var.data_force_destroy
 }

examples/data-solutions/dp-foundation/05-dtl-main.tf

@@ -82,6 +82,10 @@ module "dtl-0-prj" {
     "storage.googleapis.com",
     "storage-component.googleapis.com"
   ])
+  service_encryption_key_ids = {
+    bq      = [try(var.service_encryption_keys.bq, null)]
+    storage = [try(var.service_encryption_keys.storage, null)]
+  }
 }
 
 module "dtl-1-prj" {
@@ -107,6 +111,10 @@ module "dtl-1-prj" {
     "storage.googleapis.com",
     "storage-component.googleapis.com"
   ])
+  service_encryption_key_ids = {
+    bq      = [try(var.service_encryption_keys.bq, null)]
+    storage = [try(var.service_encryption_keys.storage, null)]
+  }
 }
 
 module "dtl-2-prj" {
@@ -132,6 +140,10 @@ module "dtl-2-prj" {
     "storage.googleapis.com",
     "storage-component.googleapis.com"
   ])
+  service_encryption_key_ids = {
+    bq      = [try(var.service_encryption_keys.bq, null)]
+    storage = [try(var.service_encryption_keys.storage, null)]
+  }
 }
 
 module "dtl-exp-prj" {
@@ -157,4 +169,8 @@ module "dtl-exp-prj" {
     "storage.googleapis.com",
     "storage-component.googleapis.com"
   ])
+  service_encryption_key_ids = {
+    bq      = [try(var.service_encryption_keys.bq, null)]
+    storage = [try(var.service_encryption_keys.storage, null)]
+  }
 }

examples/data-solutions/dp-foundation/99-kms-main.tf

@@ -12,66 +12,68 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-module "kms" {
-  count      = var.cmek_encryption ? 1 : 0
-  source     = "../../../modules/kms"
-  project_id = module.lnd-prj.project_id
-  keyring = {
-    name     = "${var.prefix}-keyring",
-    location = var.region
-  }
-  keys = {
-    key-bq  = null
-    key-cmp = null
-    key-df  = null
-    key-gcs = null
-    key-ps  = null
-  }
-  key_iam = {
-    key-bq = {
-      "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
-        "serviceAccount:${module.lnd-prj.service_accounts.robots.bq}",
-        "serviceAccount:${module.dtl-0-prj.service_accounts.robots.bq}",
-        "serviceAccount:${module.dtl-1-prj.service_accounts.robots.bq}",
-        "serviceAccount:${module.dtl-2-prj.service_accounts.robots.bq}",
-        "serviceAccount:${module.dtl-exp-prj.service_accounts.robots.bq}",
-      ]
-    },
-    key-cmp = {
-      "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
-        "serviceAccount:${module.orc-prj.service_accounts.robots.artifactregistry}",
-        "serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}",
-        "serviceAccount:${module.orc-prj.service_accounts.robots.compute}",
-        "serviceAccount:${module.orc-prj.service_accounts.robots.composer}",
-        "serviceAccount:${module.orc-prj.service_accounts.robots.pubsub}",
-        "serviceAccount:${module.orc-prj.service_accounts.robots.storage}",
+# Uncomment this section and assigne key links accondingly if you want 
+# to create a project with KMS and KMS keys
+# module "kms" {
+#   count      = var.service_encryption_keys? 1 : 0
+#   source     = "../../../modules/kms"
+#   project_id = module.lnd-prj.project_id
+#   keyring = {
+#     name     = "${var.prefix}-keyring",
+#     location = var.region
+#   }
+#   keys = {
+#     key-bq  = null
+#     key-cmp = null
+#     key-df  = null
+#     key-gcs = null
+#     key-ps  = null
+#   }
+#   key_iam = {
+#     key-bq = {
+#       "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
+#         "serviceAccount:${module.lnd-prj.service_accounts.robots.bq}",
+#         "serviceAccount:${module.dtl-0-prj.service_accounts.robots.bq}",
+#         "serviceAccount:${module.dtl-1-prj.service_accounts.robots.bq}",
+#         "serviceAccount:${module.dtl-2-prj.service_accounts.robots.bq}",
+#         "serviceAccount:${module.dtl-exp-prj.service_accounts.robots.bq}",
+#       ]
+#     },
+#     key-cmp = {
+#       "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
+#         "serviceAccount:${module.orc-prj.service_accounts.robots.artifactregistry}",
+#         "serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}",
+#         "serviceAccount:${module.orc-prj.service_accounts.robots.compute}",
+#         "serviceAccount:${module.orc-prj.service_accounts.robots.composer}",
+#         "serviceAccount:${module.orc-prj.service_accounts.robots.pubsub}",
+#         "serviceAccount:${module.orc-prj.service_accounts.robots.storage}",
 
-      ]
-    },
-    key-df = {
-      "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
-        "serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
-        "serviceAccount:${module.lod-prj.service_accounts.robots.compute}",
-        "serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}",
-        "serviceAccount:${module.trf-prj.service_accounts.robots.compute}",
-      ]
-    }
-    key-gcs = {
-      "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
-        "serviceAccount:${module.dtl-0-prj.service_accounts.robots.storage}",
-        "serviceAccount:${module.dtl-1-prj.service_accounts.robots.storage}",
-        "serviceAccount:${module.dtl-2-prj.service_accounts.robots.storage}",
-        "serviceAccount:${module.dtl-exp-prj.service_accounts.robots.storage}",
-        "serviceAccount:${module.lnd-prj.service_accounts.robots.storage}",
-        "serviceAccount:${module.lod-prj.service_accounts.robots.storage}",
-        "serviceAccount:${module.orc-prj.service_accounts.robots.storage}",
-        "serviceAccount:${module.trf-prj.service_accounts.robots.storage}",
-      ]
-    },
-    key-ps = {
-      "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
-        "serviceAccount:${module.lnd-prj.service_accounts.robots.pubsub}"
-      ]
-    }
-  }
-}
+#       ]
+#     },
+#     key-df = {
+#       "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
+#         "serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
+#         "serviceAccount:${module.lod-prj.service_accounts.robots.compute}",
+#         "serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}",
+#         "serviceAccount:${module.trf-prj.service_accounts.robots.compute}",
+#       ]
+#     }
+#     key-gcs = {
+#       "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
+#         "serviceAccount:${module.dtl-0-prj.service_accounts.robots.storage}",
+#         "serviceAccount:${module.dtl-1-prj.service_accounts.robots.storage}",
+#         "serviceAccount:${module.dtl-2-prj.service_accounts.robots.storage}",
+#         "serviceAccount:${module.dtl-exp-prj.service_accounts.robots.storage}",
+#         "serviceAccount:${module.lnd-prj.service_accounts.robots.storage}",
+#         "serviceAccount:${module.lod-prj.service_accounts.robots.storage}",
+#         "serviceAccount:${module.orc-prj.service_accounts.robots.storage}",
+#         "serviceAccount:${module.trf-prj.service_accounts.robots.storage}",
+#       ]
+#     },
+#     key-ps = {
+#       "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
+#         "serviceAccount:${module.lnd-prj.service_accounts.robots.pubsub}"
+#       ]
+#     }
+#   }
+# }

examples/data-solutions/dp-foundation/demo/README.md

@@ -0,0 +1 @@
+gsutil 

examples/data-solutions/dp-foundation/variables.tf

@@ -12,10 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-variable "cmek_encryption" {
-  description = "Flag to enable CMEK on GCP resources created."
-  type        = bool
-  default     = false
+variable "service_encryption_keys" { # service encription key
+  description = "Cloud KMS to use to encrypt different services. Key location should match service region."
+  type = object({
+    bq       = string
+    composer = string
+    dataflow = string
+    storage  = string
+    pubsub   = string
+  })
+  default = null
 }
 
 variable "composer_config" {