From f68e423c4883f50fe6f34a397dce8c96fb0b2504 Mon Sep 17 00:00:00 2001
From: Ludovico Magnocavallo <ludomagno@google.com>
Date: Fri, 14 Jun 2024 16:19:57 +0200
Subject: [PATCH] Relax dataproc master config type (#2364)

---
 modules/dataproc/README.md    | 2 +-
 modules/dataproc/variables.tf | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md
index d3aa303a..2ae54a23 100644
--- a/modules/dataproc/README.md
+++ b/modules/dataproc/README.md
@@ -260,7 +260,7 @@ module "processing-dp-cluster" {
 | [name](variables.tf#L191) | Cluster name. | <code>string</code> | ✓ |  |
 | [project_id](variables.tf#L196) | Project ID. | <code>string</code> | ✓ |  |
 | [region](variables.tf#L201) | Dataproc region. | <code>string</code> | ✓ |  |
-| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | <code title="object&#40;&#123;&#10;  graceful_decommission_timeout &#61; optional&#40;string&#41;&#10;  cluster_config &#61; optional&#40;object&#40;&#123;&#10;    staging_bucket &#61; optional&#40;string&#41;&#10;    temp_bucket    &#61; optional&#40;string&#41;&#10;    gce_cluster_config &#61; optional&#40;object&#40;&#123;&#10;      zone                   &#61; optional&#40;string&#41;&#10;      network                &#61; optional&#40;string&#41;&#10;      subnetwork             &#61; optional&#40;string&#41;&#10;      service_account        &#61; optional&#40;string&#41;&#10;      service_account_scopes &#61; optional&#40;list&#40;string&#41;&#41;&#10;      tags                   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;      internal_ip_only       &#61; optional&#40;bool&#41;&#10;      metadata               &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;      reservation_affinity &#61; optional&#40;object&#40;&#123;&#10;        consume_reservation_type &#61; string&#10;        key                      &#61; string&#10;        values                   &#61; string&#10;      &#125;&#41;&#41;&#10;      node_group_affinity &#61; optional&#40;object&#40;&#123;&#10;        node_group_uri &#61; string&#10;      &#125;&#41;&#41;&#10;&#10;&#10;      shielded_instance_config &#61; optional&#40;object&#40;&#123;&#10;        enable_secure_boot          &#61; bool&#10;        enable_vtpm                 &#61; bool&#10;        enable_integrity_monitoring &#61; bool&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    master_config &#61; optional&#40;object&#40;&#123;&#10;      num_instances    &#61; number&#10;      machine_type     &#61; string&#10;      min_cpu_platform &#61; string&#10;      image_uri        &#61; string&#10;      disk_config &#61; optional&#40;object&#40;&#123;&#10;        boot_disk_type    &#61; string&#10;        boot_disk_size_gb &#61; number&#10;        num_local_ssds    &#61; number&#10;      &#125;&#41;&#41;&#10;      accelerators &#61; optional&#40;object&#40;&#123;&#10;        accelerator_type  &#61; string&#10;        accelerator_count &#61; number&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    worker_config &#61; optional&#40;object&#40;&#123;&#10;      num_instances    &#61; number&#10;      machine_type     &#61; string&#10;      min_cpu_platform &#61; string&#10;      disk_config &#61; optional&#40;object&#40;&#123;&#10;        boot_disk_type    &#61; string&#10;        boot_disk_size_gb &#61; number&#10;        num_local_ssds    &#61; number&#10;      &#125;&#41;&#41;&#10;      image_uri &#61; string&#10;      accelerators &#61; optional&#40;object&#40;&#123;&#10;        accelerator_type  &#61; string&#10;        accelerator_count &#61; number&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    preemptible_worker_config &#61; optional&#40;object&#40;&#123;&#10;      num_instances  &#61; number&#10;      preemptibility &#61; string&#10;      disk_config &#61; optional&#40;object&#40;&#123;&#10;        boot_disk_type    &#61; string&#10;        boot_disk_size_gb &#61; number&#10;        num_local_ssds    &#61; number&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    software_config &#61; optional&#40;object&#40;&#123;&#10;      image_version       &#61; optional&#40;string&#41;&#10;      override_properties &#61; map&#40;string&#41;&#10;      optional_components &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    security_config &#61; optional&#40;object&#40;&#123;&#10;      kerberos_config &#61; object&#40;&#123;&#10;        cross_realm_trust_admin_server        &#61; optional&#40;string&#41;&#10;        cross_realm_trust_kdc                 &#61; optional&#40;string&#41;&#10;        cross_realm_trust_realm               &#61; optional&#40;string&#41;&#10;        cross_realm_trust_shared_password_uri &#61; optional&#40;string&#41;&#10;        enable_kerberos                       &#61; optional&#40;string&#41;&#10;        kdc_db_key_uri                        &#61; optional&#40;string&#41;&#10;        key_password_uri                      &#61; optional&#40;string&#41;&#10;        keystore_uri                          &#61; optional&#40;string&#41;&#10;        keystore_password_uri                 &#61; optional&#40;string&#41;&#10;        kms_key_uri                           &#61; string&#10;        realm                                 &#61; optional&#40;string&#41;&#10;        root_principal_password_uri           &#61; string&#10;        tgt_lifetime_hours                    &#61; optional&#40;string&#41;&#10;        truststore_password_uri               &#61; optional&#40;string&#41;&#10;        truststore_uri                        &#61; optional&#40;string&#41;&#10;      &#125;&#41;&#10;    &#125;&#41;&#41;&#10;    autoscaling_config &#61; optional&#40;object&#40;&#123;&#10;      policy_uri &#61; string&#10;    &#125;&#41;&#41;&#10;    initialization_action &#61; optional&#40;object&#40;&#123;&#10;      script      &#61; string&#10;      timeout_sec &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#41;&#10;    encryption_config &#61; optional&#40;object&#40;&#123;&#10;      kms_key_name &#61; string&#10;    &#125;&#41;&#41;&#10;    lifecycle_config &#61; optional&#40;object&#40;&#123;&#10;      idle_delete_ttl  &#61; optional&#40;string&#41;&#10;      auto_delete_time &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#41;&#10;    endpoint_config &#61; optional&#40;object&#40;&#123;&#10;      enable_http_port_access &#61; string&#10;    &#125;&#41;&#41;&#10;    dataproc_metric_config &#61; optional&#40;object&#40;&#123;&#10;      metrics &#61; list&#40;object&#40;&#123;&#10;        metric_source    &#61; string&#10;        metric_overrides &#61; optional&#40;list&#40;string&#41;&#41;&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    metastore_config &#61; optional&#40;object&#40;&#123;&#10;      dataproc_metastore_service &#61; string&#10;    &#125;&#41;&#41;&#10;  &#125;&#41;&#41;&#10;&#10;&#10;  virtual_cluster_config &#61; optional&#40;object&#40;&#123;&#10;    staging_bucket &#61; optional&#40;string&#41;&#10;    auxiliary_services_config &#61; optional&#40;object&#40;&#123;&#10;      metastore_config &#61; optional&#40;object&#40;&#123;&#10;        dataproc_metastore_service &#61; string&#10;      &#125;&#41;&#41;&#10;      spark_history_server_config &#61; optional&#40;object&#40;&#123;&#10;        dataproc_cluster &#61; string&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    kubernetes_cluster_config &#61; object&#40;&#123;&#10;      kubernetes_namespace &#61; optional&#40;string&#41;&#10;      kubernetes_software_config &#61; object&#40;&#123;&#10;        component_version &#61; map&#40;string&#41;&#10;        properties        &#61; optional&#40;map&#40;string&#41;&#41;&#10;      &#125;&#41;&#10;&#10;&#10;      gke_cluster_config &#61; object&#40;&#123;&#10;        gke_cluster_target &#61; optional&#40;string&#41;&#10;        node_pool_target &#61; optional&#40;object&#40;&#123;&#10;          node_pool &#61; string&#10;          roles     &#61; list&#40;string&#41;&#10;          node_pool_config &#61; optional&#40;object&#40;&#123;&#10;            autoscaling &#61; optional&#40;object&#40;&#123;&#10;              min_node_count &#61; optional&#40;number&#41;&#10;              max_node_count &#61; optional&#40;number&#41;&#10;            &#125;&#41;&#41;&#10;&#10;&#10;            config &#61; object&#40;&#123;&#10;              machine_type     &#61; optional&#40;string&#41;&#10;              preemptible      &#61; optional&#40;bool&#41;&#10;              local_ssd_count  &#61; optional&#40;number&#41;&#10;              min_cpu_platform &#61; optional&#40;string&#41;&#10;              spot             &#61; optional&#40;bool&#41;&#10;            &#125;&#41;&#10;&#10;&#10;            locations &#61; optional&#40;list&#40;string&#41;&#41;&#10;          &#125;&#41;&#41;&#10;        &#125;&#41;&#41;&#10;      &#125;&#41;&#10;    &#125;&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | <code title="object&#40;&#123;&#10;  graceful_decommission_timeout &#61; optional&#40;string&#41;&#10;  cluster_config &#61; optional&#40;object&#40;&#123;&#10;    staging_bucket &#61; optional&#40;string&#41;&#10;    temp_bucket    &#61; optional&#40;string&#41;&#10;    gce_cluster_config &#61; optional&#40;object&#40;&#123;&#10;      zone                   &#61; optional&#40;string&#41;&#10;      network                &#61; optional&#40;string&#41;&#10;      subnetwork             &#61; optional&#40;string&#41;&#10;      service_account        &#61; optional&#40;string&#41;&#10;      service_account_scopes &#61; optional&#40;list&#40;string&#41;&#41;&#10;      tags                   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;      internal_ip_only       &#61; optional&#40;bool&#41;&#10;      metadata               &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;      reservation_affinity &#61; optional&#40;object&#40;&#123;&#10;        consume_reservation_type &#61; string&#10;        key                      &#61; string&#10;        values                   &#61; string&#10;      &#125;&#41;&#41;&#10;      node_group_affinity &#61; optional&#40;object&#40;&#123;&#10;        node_group_uri &#61; string&#10;      &#125;&#41;&#41;&#10;&#10;&#10;      shielded_instance_config &#61; optional&#40;object&#40;&#123;&#10;        enable_secure_boot          &#61; bool&#10;        enable_vtpm                 &#61; bool&#10;        enable_integrity_monitoring &#61; bool&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    master_config &#61; optional&#40;object&#40;&#123;&#10;      num_instances    &#61; number&#10;      machine_type     &#61; optional&#40;string&#41;&#10;      min_cpu_platform &#61; optional&#40;string&#41;&#10;      image_uri        &#61; optional&#40;string&#41;&#10;      disk_config &#61; optional&#40;object&#40;&#123;&#10;        boot_disk_type    &#61; string&#10;        boot_disk_size_gb &#61; number&#10;        num_local_ssds    &#61; number&#10;      &#125;&#41;&#41;&#10;      accelerators &#61; optional&#40;object&#40;&#123;&#10;        accelerator_type  &#61; string&#10;        accelerator_count &#61; number&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    worker_config &#61; optional&#40;object&#40;&#123;&#10;      num_instances    &#61; number&#10;      machine_type     &#61; string&#10;      min_cpu_platform &#61; string&#10;      disk_config &#61; optional&#40;object&#40;&#123;&#10;        boot_disk_type    &#61; string&#10;        boot_disk_size_gb &#61; number&#10;        num_local_ssds    &#61; number&#10;      &#125;&#41;&#41;&#10;      image_uri &#61; string&#10;      accelerators &#61; optional&#40;object&#40;&#123;&#10;        accelerator_type  &#61; string&#10;        accelerator_count &#61; number&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    preemptible_worker_config &#61; optional&#40;object&#40;&#123;&#10;      num_instances  &#61; number&#10;      preemptibility &#61; string&#10;      disk_config &#61; optional&#40;object&#40;&#123;&#10;        boot_disk_type    &#61; string&#10;        boot_disk_size_gb &#61; number&#10;        num_local_ssds    &#61; number&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    software_config &#61; optional&#40;object&#40;&#123;&#10;      image_version       &#61; optional&#40;string&#41;&#10;      override_properties &#61; map&#40;string&#41;&#10;      optional_components &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    security_config &#61; optional&#40;object&#40;&#123;&#10;      kerberos_config &#61; object&#40;&#123;&#10;        cross_realm_trust_admin_server        &#61; optional&#40;string&#41;&#10;        cross_realm_trust_kdc                 &#61; optional&#40;string&#41;&#10;        cross_realm_trust_realm               &#61; optional&#40;string&#41;&#10;        cross_realm_trust_shared_password_uri &#61; optional&#40;string&#41;&#10;        enable_kerberos                       &#61; optional&#40;string&#41;&#10;        kdc_db_key_uri                        &#61; optional&#40;string&#41;&#10;        key_password_uri                      &#61; optional&#40;string&#41;&#10;        keystore_uri                          &#61; optional&#40;string&#41;&#10;        keystore_password_uri                 &#61; optional&#40;string&#41;&#10;        kms_key_uri                           &#61; string&#10;        realm                                 &#61; optional&#40;string&#41;&#10;        root_principal_password_uri           &#61; string&#10;        tgt_lifetime_hours                    &#61; optional&#40;string&#41;&#10;        truststore_password_uri               &#61; optional&#40;string&#41;&#10;        truststore_uri                        &#61; optional&#40;string&#41;&#10;      &#125;&#41;&#10;    &#125;&#41;&#41;&#10;    autoscaling_config &#61; optional&#40;object&#40;&#123;&#10;      policy_uri &#61; string&#10;    &#125;&#41;&#41;&#10;    initialization_action &#61; optional&#40;object&#40;&#123;&#10;      script      &#61; string&#10;      timeout_sec &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#41;&#10;    encryption_config &#61; optional&#40;object&#40;&#123;&#10;      kms_key_name &#61; string&#10;    &#125;&#41;&#41;&#10;    lifecycle_config &#61; optional&#40;object&#40;&#123;&#10;      idle_delete_ttl  &#61; optional&#40;string&#41;&#10;      auto_delete_time &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#41;&#10;    endpoint_config &#61; optional&#40;object&#40;&#123;&#10;      enable_http_port_access &#61; string&#10;    &#125;&#41;&#41;&#10;    dataproc_metric_config &#61; optional&#40;object&#40;&#123;&#10;      metrics &#61; list&#40;object&#40;&#123;&#10;        metric_source    &#61; string&#10;        metric_overrides &#61; optional&#40;list&#40;string&#41;&#41;&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    metastore_config &#61; optional&#40;object&#40;&#123;&#10;      dataproc_metastore_service &#61; string&#10;    &#125;&#41;&#41;&#10;  &#125;&#41;&#41;&#10;&#10;&#10;  virtual_cluster_config &#61; optional&#40;object&#40;&#123;&#10;    staging_bucket &#61; optional&#40;string&#41;&#10;    auxiliary_services_config &#61; optional&#40;object&#40;&#123;&#10;      metastore_config &#61; optional&#40;object&#40;&#123;&#10;        dataproc_metastore_service &#61; string&#10;      &#125;&#41;&#41;&#10;      spark_history_server_config &#61; optional&#40;object&#40;&#123;&#10;        dataproc_cluster &#61; string&#10;      &#125;&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    kubernetes_cluster_config &#61; object&#40;&#123;&#10;      kubernetes_namespace &#61; optional&#40;string&#41;&#10;      kubernetes_software_config &#61; object&#40;&#123;&#10;        component_version &#61; map&#40;string&#41;&#10;        properties        &#61; optional&#40;map&#40;string&#41;&#41;&#10;      &#125;&#41;&#10;&#10;&#10;      gke_cluster_config &#61; object&#40;&#123;&#10;        gke_cluster_target &#61; optional&#40;string&#41;&#10;        node_pool_target &#61; optional&#40;object&#40;&#123;&#10;          node_pool &#61; string&#10;          roles     &#61; list&#40;string&#41;&#10;          node_pool_config &#61; optional&#40;object&#40;&#123;&#10;            autoscaling &#61; optional&#40;object&#40;&#123;&#10;              min_node_count &#61; optional&#40;number&#41;&#10;              max_node_count &#61; optional&#40;number&#41;&#10;            &#125;&#41;&#41;&#10;&#10;&#10;            config &#61; object&#40;&#123;&#10;              machine_type     &#61; optional&#40;string&#41;&#10;              preemptible      &#61; optional&#40;bool&#41;&#10;              local_ssd_count  &#61; optional&#40;number&#41;&#10;              min_cpu_platform &#61; optional&#40;string&#41;&#10;              spot             &#61; optional&#40;bool&#41;&#10;            &#125;&#41;&#10;&#10;&#10;            locations &#61; optional&#40;list&#40;string&#41;&#41;&#10;          &#125;&#41;&#41;&#10;        &#125;&#41;&#41;&#10;      &#125;&#41;&#10;    &#125;&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [iam](variables-iam.tf#L24) | IAM bindings in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [iam_bindings](variables-iam.tf#L31) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10;  members &#61; list&#40;string&#41;&#10;  role    &#61; string&#10;  condition &#61; optional&#40;object&#40;&#123;&#10;    expression  &#61; string&#10;    title       &#61; string&#10;    description &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [iam_bindings_additive](variables-iam.tf#L46) | Individual additive IAM bindings. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10;  member &#61; string&#10;  role   &#61; string&#10;  condition &#61; optional&#40;object&#40;&#123;&#10;    expression  &#61; string&#10;    title       &#61; string&#10;    description &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf
index a8164aaf..371c6b44 100644
--- a/modules/dataproc/variables.tf
+++ b/modules/dataproc/variables.tf
@@ -47,9 +47,9 @@ variable "dataproc_config" {
       }))
       master_config = optional(object({
         num_instances    = number
-        machine_type     = string
-        min_cpu_platform = string
-        image_uri        = string
+        machine_type     = optional(string)
+        min_cpu_platform = optional(string)
+        image_uri        = optional(string)
         disk_config = optional(object({
           boot_disk_type    = string
           boot_disk_size_gb = number