diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md
index 80835dd1..d071ecda 100644
--- a/modules/dataproc/README.md
+++ b/modules/dataproc/README.md
@@ -46,6 +46,35 @@ module "processing-dp-cluster" {
# tftest modules=1 resources=1
```
+### Cluster with CMEK encryption
+
+To set cluster configuration use the Customer Managed Encryption key, set `dataproc_config.encryption_config.` variable. The Compute Engine service agent and the Cloud Storage service agent need to have `CryptoKey Encrypter/Decrypter` role on they configured KMS key ([Documentation](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/customer-managed-encryption)).
+
+```hcl
+module "processing-dp-cluster" {
+ source = "./fabric/modules/dataproc"
+ project_id = "my-project"
+ name = "my-cluster"
+ region = "europe-west1"
+ prefix = "prefix"
+ dataproc_config = {
+ cluster_config = {
+ gce_cluster_config = {
+ subnetwork = "https://www.googleapis.com/compute/v1/projects/PROJECT/regions/europe-west1/subnetworks/SUBNET"
+ zone = "europe-west1-b"
+ service_account = ""
+ service_account_scopes = ["cloud-platform"]
+ internal_ip_only = true
+ }
+ }
+ encryption_config = {
+ kms_key_name = "projects/project-id/locations/region/keyRings/key-ring-name/cryptoKeys/key-name"
+ }
+ }
+}
+# tftest modules=1 resources=1
+```
+
## IAM Examples
IAM is managed via several variables that implement different levels of control:
@@ -119,7 +148,7 @@ module "processing-dp-cluster" {
| [name](variables.tf#L211) | Cluster name. | string
| ✓ | |
| [project_id](variables.tf#L226) | Project ID. | string
| ✓ | |
| [region](variables.tf#L231) | Dataproc region. | string
| ✓ | |
-| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…})
| | {}
|
+| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…})
| | {}
|
| [group_iam](variables.tf#L184) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string))
| | {}
|
| [iam](variables.tf#L191) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
| [iam_additive](variables.tf#L198) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
diff --git a/modules/dataproc/main.tf b/modules/dataproc/main.tf
index ab09cbea..55bef5c7 100644
--- a/modules/dataproc/main.tf
+++ b/modules/dataproc/main.tf
@@ -59,9 +59,9 @@ resource "google_dataproc_cluster" "cluster" {
dynamic "shielded_instance_config" {
for_each = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config == null ? [] : [""]
content {
- enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_secure_boot
- enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_vtpm
- enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_integrity_monitoring
+ enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_secure_boot
+ enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_vtpm
+ enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_integrity_monitoring
}
}
}
@@ -99,9 +99,9 @@ resource "google_dataproc_cluster" "cluster" {
dynamic "disk_config" {
for_each = var.dataproc_config.cluster_config.worker_config.disk_config == null ? [] : [""]
content {
- boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_type
- boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_size_gb
- num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.value.num_local_ssds
+ boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.boot_disk_type
+ boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.boot_disk_size_gb
+ num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.num_local_ssds
}
}
image_uri = var.dataproc_config.cluster_config.worker_config.image_uri
@@ -165,20 +165,20 @@ resource "google_dataproc_cluster" "cluster" {
dynamic "autoscaling_config" {
for_each = var.dataproc_config.cluster_config.autoscaling_config == null ? [] : [""]
content {
- policy_uri = var.dataproc_config.cluster_config.autoscaling_config.value.policy_uri
+ policy_uri = var.dataproc_config.cluster_config.autoscaling_config.policy_uri
}
}
dynamic "initialization_action" {
for_each = var.dataproc_config.cluster_config.initialization_action == null ? [] : [""]
content {
- script = var.dataproc_config.cluster_config.initialization_action.value.script
- timeout_sec = var.dataproc_config.cluster_config.initialization_action.value.timeout_sec
+ script = var.dataproc_config.cluster_config.initialization_action.script
+ timeout_sec = var.dataproc_config.cluster_config.initialization_action.timeout_sec
}
}
dynamic "encryption_config" {
- for_each = var.dataproc_config.cluster_config.encryption_config == null ? [] : [""]
+ for_each = try(var.dataproc_config.cluster_config.encryption_config.kms_key_name == null ? [] : [""], [])
content {
- kms_key_name = var.dataproc_config.cluster_config.encryption_config.value.kms_key_name
+ kms_key_name = var.dataproc_config.cluster_config.encryption_config.kms_key_name
}
}
dynamic "dataproc_metric_config" {
@@ -243,8 +243,8 @@ resource "google_dataproc_cluster" "cluster" {
dynamic "kubernetes_software_config" {
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config == null ? [] : [""]
content {
- component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.component_version
- properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.properties
+ component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.component_version
+ properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.properties
}
}
diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf
index 3636a706..314d2431 100644
--- a/modules/dataproc/variables.tf
+++ b/modules/dataproc/variables.tf
@@ -84,9 +84,9 @@ variable "dataproc_config" {
}), null)
}), null)
software_config = optional(object({
- image_version = string
- override_properties = list(map(string))
- optional_components = list(string)
+ image_version = optional(string, null)
+ override_properties = map(string)
+ optional_components = optional(list(string), null)
}), null)
security_config = optional(object({
kerberos_config = object({