diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 80835dd1..d071ecda 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -46,6 +46,35 @@ module "processing-dp-cluster" { # tftest modules=1 resources=1 ``` +### Cluster with CMEK encryption + +To set cluster configuration use the Customer Managed Encryption key, set `dataproc_config.encryption_config.` variable. The Compute Engine service agent and the Cloud Storage service agent need to have `CryptoKey Encrypter/Decrypter` role on they configured KMS key ([Documentation](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/customer-managed-encryption)). + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + dataproc_config = { + cluster_config = { + gce_cluster_config = { + subnetwork = "https://www.googleapis.com/compute/v1/projects/PROJECT/regions/europe-west1/subnetworks/SUBNET" + zone = "europe-west1-b" + service_account = "" + service_account_scopes = ["cloud-platform"] + internal_ip_only = true + } + } + encryption_config = { + kms_key_name = "projects/project-id/locations/region/keyRings/key-ring-name/cryptoKeys/key-name" + } + } +} +# tftest modules=1 resources=1 +``` + ## IAM Examples IAM is managed via several variables that implement different levels of control: @@ -119,7 +148,7 @@ module "processing-dp-cluster" { | [name](variables.tf#L211) | Cluster name. | string | ✓ | | | [project_id](variables.tf#L226) | Project ID. | string | ✓ | | | [region](variables.tf#L231) | Dataproc region. | string | ✓ | | -| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | +| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | | [group_iam](variables.tf#L184) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L191) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L198) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | diff --git a/modules/dataproc/main.tf b/modules/dataproc/main.tf index ab09cbea..55bef5c7 100644 --- a/modules/dataproc/main.tf +++ b/modules/dataproc/main.tf @@ -59,9 +59,9 @@ resource "google_dataproc_cluster" "cluster" { dynamic "shielded_instance_config" { for_each = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config == null ? [] : [""] content { - enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_secure_boot - enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_vtpm - enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_integrity_monitoring + enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_secure_boot + enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_vtpm + enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_integrity_monitoring } } } @@ -99,9 +99,9 @@ resource "google_dataproc_cluster" "cluster" { dynamic "disk_config" { for_each = var.dataproc_config.cluster_config.worker_config.disk_config == null ? [] : [""] content { - boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_type - boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_size_gb - num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.value.num_local_ssds + boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.boot_disk_type + boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.boot_disk_size_gb + num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.num_local_ssds } } image_uri = var.dataproc_config.cluster_config.worker_config.image_uri @@ -165,20 +165,20 @@ resource "google_dataproc_cluster" "cluster" { dynamic "autoscaling_config" { for_each = var.dataproc_config.cluster_config.autoscaling_config == null ? [] : [""] content { - policy_uri = var.dataproc_config.cluster_config.autoscaling_config.value.policy_uri + policy_uri = var.dataproc_config.cluster_config.autoscaling_config.policy_uri } } dynamic "initialization_action" { for_each = var.dataproc_config.cluster_config.initialization_action == null ? [] : [""] content { - script = var.dataproc_config.cluster_config.initialization_action.value.script - timeout_sec = var.dataproc_config.cluster_config.initialization_action.value.timeout_sec + script = var.dataproc_config.cluster_config.initialization_action.script + timeout_sec = var.dataproc_config.cluster_config.initialization_action.timeout_sec } } dynamic "encryption_config" { - for_each = var.dataproc_config.cluster_config.encryption_config == null ? [] : [""] + for_each = try(var.dataproc_config.cluster_config.encryption_config.kms_key_name == null ? [] : [""], []) content { - kms_key_name = var.dataproc_config.cluster_config.encryption_config.value.kms_key_name + kms_key_name = var.dataproc_config.cluster_config.encryption_config.kms_key_name } } dynamic "dataproc_metric_config" { @@ -243,8 +243,8 @@ resource "google_dataproc_cluster" "cluster" { dynamic "kubernetes_software_config" { for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config == null ? [] : [""] content { - component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.component_version - properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.properties + component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.component_version + properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.properties } } diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index 3636a706..314d2431 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -84,9 +84,9 @@ variable "dataproc_config" { }), null) }), null) software_config = optional(object({ - image_version = string - override_properties = list(map(string)) - optional_components = list(string) + image_version = optional(string, null) + override_properties = map(string) + optional_components = optional(list(string), null) }), null) security_config = optional(object({ kerberos_config = object({