diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md new file mode 100644 index 00000000..b7afb9f9 --- /dev/null +++ b/modules/dataproc/README.md @@ -0,0 +1,131 @@ +# Google Cloud Dataproc + +This module Manages a Google ['Cloud Dataproc'](https://cloud.google.com/dataproc) cluster resource, including IAM. + +## TODO + +- [ ] Add support for Cloud Dataproc ['autoscaling policy'](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/dataproc_autoscaling_policy_iam). + +## Examples + +### Simple + +```hcl +module "processing-dp-cluster-2" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" +} +# tftest modules=1 resources=1 +``` + +### Cluster configuration + +To set cluster configuration use the 'dataproc_config.cluster_config' variable. + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + dataproc_config = { + cluster_config = { + gce_cluster_config = { + subnetwork = "https://www.googleapis.com/compute/v1/projects/PROJECT/regions/europe-west1/subnetworks/SUBNET" + zone = "europe-west1-b" + service_account = "" + service_account_scopes = ["cloud-platform"] + internal_ip_only = true + } + } + } +} +# tftest modules=1 resources=1 +``` + +## IAM Examples + +IAM is managed via several variables that implement different levels of control: + +- `group_iam` and `iam` configure authoritative bindings that manage individual roles exclusively, mapping to the `google_dataproc_cluster_iam_binding` resource +- `iam_additive` configure additive bindings that only manage individual role/member pairs, mapping to the `google_dataproc_cluster_iam_member` resource + +### Authorative IAM + +The iam variable is based on role keys and is typically used for service accounts, or where member values can be dynamic and would create potential problems in the underlying for_each cycle. + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + iam = { + "roles/dataproc.viewer" = [ + "serviceAccount:service-account@PROJECT_ID.iam.gserviceaccount.com" + ] + } +} +# tftest modules=1 resources=2 +``` + +The group_iam variable uses group email addresses as keys and is a convenient way to assign roles to humans following Google's best practices. The end result is readable code that also serves as documentation. + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + group_iam = { + "gcp-data-engineers@example.net" = [ + "roles/dataproc.viewer" + ] + } +} +# tftest modules=1 resources=2 +``` + +### Additive IAM + +Additive IAM is typically used where bindings for specific roles are controlled by different modules or in different Terraform stages. One example is when the cluster is created by one team but a different team manages access. + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + iam_additive = { + "roles/dataproc.viewer" = [ + "serviceAccount:service-account@PROJECT_ID.iam.gserviceaccount.com" + ] + } +} +# tftest modules=1 resources=2 +``` + + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L211) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L226) | Project ID. | string | ✓ | | +| [region](variables.tf#L231) | Dataproc region. | string | ✓ | | +| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | +| [group_iam](variables.tf#L184) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L191) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive](variables.tf#L198) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [labels](variables.tf#L205) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [prefix](variables.tf#L216) | Optional prefix used to generate project id and name. | string | | null | +| [service_account](variables.tf#L236) | Service account to set on the Dataproc cluster. | string | | null | + + diff --git a/modules/dataproc/iam.tf b/modules/dataproc/iam.tf new file mode 100644 index 00000000..e44c2a63 --- /dev/null +++ b/modules/dataproc/iam.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# # tfdoc:file:description Generic IAM bindings and roles. + +locals { + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + _iam_additive_pairs = flatten([ + for role, members in var.iam_additive : [ + for member in members : { role = role, member = member } + ] + ]) + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } + iam_additive = { + for pair in local._iam_additive_pairs : + "${pair.role}-${pair.member}" => { + role = pair.role + member = pair.member + } + } +} + +resource "google_dataproc_cluster_iam_binding" "authoritative" { + for_each = local.iam + project = var.project_id + cluster = google_dataproc_cluster.cluster.name + region = var.region + role = each.key + members = each.value +} + +resource "google_dataproc_cluster_iam_member" "additive" { + for_each = ( + length(var.iam_additive) > 0 + ? local.iam_additive + : {} + ) + project = var.project_id + cluster = google_dataproc_cluster.cluster.name + region = var.region + role = each.value.role + member = each.value.member +} diff --git a/modules/dataproc/main.tf b/modules/dataproc/main.tf new file mode 100644 index 00000000..ab09cbea --- /dev/null +++ b/modules/dataproc/main.tf @@ -0,0 +1,298 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Cloud Dataproc resource definition. + +locals { + prefix = var.prefix == null ? "" : "${var.prefix}-" +} + +resource "google_dataproc_cluster" "cluster" { + name = "${local.prefix}${var.name}" + project = var.project_id + region = var.region + graceful_decommission_timeout = var.dataproc_config.graceful_decommission_timeout + labels = var.labels + dynamic "cluster_config" { + for_each = var.dataproc_config.cluster_config == null ? [] : [""] + content { + staging_bucket = var.dataproc_config.cluster_config.staging_bucket + temp_bucket = var.dataproc_config.cluster_config.temp_bucket + dynamic "gce_cluster_config" { + for_each = var.dataproc_config.cluster_config.gce_cluster_config == null ? [] : [""] + content { + zone = var.dataproc_config.cluster_config.gce_cluster_config.zone + network = var.dataproc_config.cluster_config.gce_cluster_config.network + subnetwork = var.dataproc_config.cluster_config.gce_cluster_config.subnetwork + service_account = var.dataproc_config.cluster_config.gce_cluster_config.service_account + service_account_scopes = var.dataproc_config.cluster_config.gce_cluster_config.service_account_scopes + tags = var.dataproc_config.cluster_config.gce_cluster_config.tags + internal_ip_only = var.dataproc_config.cluster_config.gce_cluster_config.internal_ip_only + metadata = var.dataproc_config.cluster_config.gce_cluster_config.metadata + dynamic "reservation_affinity" { + for_each = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity == null ? [] : [""] + content { + consume_reservation_type = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.consume_reservation_type + key = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.key + values = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.value + } + } + dynamic "node_group_affinity" { + for_each = var.dataproc_config.cluster_config.gce_cluster_config.node_group_affinity == null ? [] : [""] + content { + node_group_uri = var.dataproc_config.cluster_config.gce_cluster_config.node_group_uri + } + } + dynamic "shielded_instance_config" { + for_each = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config == null ? [] : [""] + content { + enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_secure_boot + enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_vtpm + enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_integrity_monitoring + } + } + } + } + dynamic "master_config" { + for_each = var.dataproc_config.cluster_config.master_config == null ? [] : [""] + content { + num_instances = var.dataproc_config.cluster_config.master_config.num_instances + machine_type = var.dataproc_config.cluster_config.master_config.machine_type + min_cpu_platform = var.dataproc_config.cluster_config.master_config.min_cpu_platform + image_uri = var.dataproc_config.cluster_config.master_config.image_uri + dynamic "disk_config" { + for_each = var.dataproc_config.cluster_config.master_config.disk_config == null ? [] : [""] + content { + boot_disk_type = var.dataproc_config.cluster_config.master_config.disk_config.boot_disk_type + boot_disk_size_gb = var.dataproc_config.cluster_config.master_config.disk_config.boot_disk_size_gb + num_local_ssds = var.dataproc_config.cluster_config.master_config.disk_config.num_local_ssds + } + } + dynamic "accelerators" { + for_each = var.dataproc_config.cluster_config.master_config.accelerators == null ? [] : [""] + content { + accelerator_type = var.dataproc_config.cluster_config.master_config.accelerators.accelerator_type + accelerator_count = var.dataproc_config.cluster_config.master_config.accelerators.accelerator_count + } + } + } + } + dynamic "worker_config" { + for_each = var.dataproc_config.cluster_config.worker_config == null ? [] : [""] + content { + num_instances = var.dataproc_config.cluster_config.worker_config.num_instances + machine_type = var.dataproc_config.cluster_config.worker_config.machine_type + min_cpu_platform = var.dataproc_config.cluster_config.worker_config.min_cpu_platform + dynamic "disk_config" { + for_each = var.dataproc_config.cluster_config.worker_config.disk_config == null ? [] : [""] + content { + boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_type + boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_size_gb + num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.value.num_local_ssds + } + } + image_uri = var.dataproc_config.cluster_config.worker_config.image_uri + dynamic "accelerators" { + for_each = var.dataproc_config.cluster_config.worker_config.accelerators == null ? [] : [""] + content { + accelerator_type = var.dataproc_config.cluster_config.accelerators.accelerator_type + accelerator_count = var.dataproc_config.cluster_config.accelerators.accelerator_count + } + } + } + } + dynamic "preemptible_worker_config" { + for_each = var.dataproc_config.cluster_config.preemptible_worker_config == null ? [] : [""] + content { + num_instances = var.dataproc_config.cluster_config.preemptible_worker_config.num_instances + preemptibility = var.dataproc_config.cluster_config.preemptible_worker_config.preemptibility + dynamic "disk_config" { + for_each = var.dataproc_config.cluster_config.preemptible_worker_config.disk_config == null ? [] : [""] + content { + boot_disk_type = var.dataproc_config.cluster_config.disk_config.boot_disk_type + boot_disk_size_gb = var.dataproc_config.cluster_config.disk_config.boot_disk_size_gb + num_local_ssds = var.dataproc_config.cluster_config.disk_config.num_local_ssds + } + } + } + } + dynamic "software_config" { + for_each = var.dataproc_config.cluster_config.software_config == null ? [] : [""] + content { + image_version = var.dataproc_config.cluster_config.software_config.image_version + override_properties = var.dataproc_config.cluster_config.software_config.override_properties + optional_components = var.dataproc_config.cluster_config.software_config.optional_components + } + } + dynamic "security_config" { + for_each = var.dataproc_config.cluster_config.security_config == null ? [] : [""] + content { + dynamic "kerberos_config" { + for_each = try(var.dataproc_config.cluster_config.security_config.kerberos_config == null ? [] : [""], []) + content { + cross_realm_trust_admin_server = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_admin_server + cross_realm_trust_kdc = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_kdc + cross_realm_trust_realm = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_realm + cross_realm_trust_shared_password_uri = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_shared_password_uri + enable_kerberos = var.dataproc_config.cluster_config.kerberos_config.enable_kerberos + kdc_db_key_uri = var.dataproc_config.cluster_config.kerberos_config.kdc_db_key_uri + key_password_uri = var.dataproc_config.cluster_config.kerberos_config.key_password_uri + keystore_uri = var.dataproc_config.cluster_config.kerberos_config.keystore_uri + keystore_password_uri = var.dataproc_config.cluster_config.kerberos_config.keystore_password_uri + kms_key_uri = var.dataproc_config.cluster_config.kerberos_config.kms_key_uri + realm = var.dataproc_config.cluster_config.kerberos_config.realm + root_principal_password_uri = var.dataproc_config.cluster_config.kerberos_config.root_principal_password_uri + tgt_lifetime_hours = var.dataproc_config.cluster_config.kerberos_config.tgt_lifetime_hours + truststore_password_uri = var.dataproc_config.cluster_config.kerberos_config.truststore_password_uri + truststore_uri = var.dataproc_config.cluster_config.kerberos_config.truststore_uri + } + } + } + } + dynamic "autoscaling_config" { + for_each = var.dataproc_config.cluster_config.autoscaling_config == null ? [] : [""] + content { + policy_uri = var.dataproc_config.cluster_config.autoscaling_config.value.policy_uri + } + } + dynamic "initialization_action" { + for_each = var.dataproc_config.cluster_config.initialization_action == null ? [] : [""] + content { + script = var.dataproc_config.cluster_config.initialization_action.value.script + timeout_sec = var.dataproc_config.cluster_config.initialization_action.value.timeout_sec + } + } + dynamic "encryption_config" { + for_each = var.dataproc_config.cluster_config.encryption_config == null ? [] : [""] + content { + kms_key_name = var.dataproc_config.cluster_config.encryption_config.value.kms_key_name + } + } + dynamic "dataproc_metric_config" { + for_each = var.dataproc_config.cluster_config.dataproc_metric_config == null ? [] : [""] + content { + dynamic "metrics" { + for_each = var.dataproc_config.cluster_config.dataproc_metric_config.metrics == null ? [] : [""] + content { + metric_source = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_source + metric_overrides = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_overrides + } + } + } + } + dynamic "lifecycle_config" { + for_each = var.dataproc_config.cluster_config.lifecycle_config == null ? [] : [""] + content { + idle_delete_ttl = var.dataproc_config.cluster_config.lifecycle_config.idle_delete_ttl + auto_delete_time = var.dataproc_config.cluster_config.lifecycle_config.auto_delete_time + } + } + dynamic "endpoint_config" { + for_each = var.dataproc_config.cluster_config.endpoint_config == null ? [] : [""] + content { + enable_http_port_access = var.dataproc_config.cluster_config.endpoint_config.enable_http_port_access + } + } + dynamic "metastore_config" { + for_each = var.dataproc_config.cluster_config.metastore_config == null ? [] : [""] + content { + dataproc_metastore_service = var.dataproc_config.cluster_config.metastore_config.dataproc_metastore_service + } + } + + } + } + + dynamic "virtual_cluster_config" { + for_each = var.dataproc_config.virtual_cluster_config == null ? [] : [""] + content { + dynamic "auxiliary_services_config" { + for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config == null ? [] : [""] + content { + dynamic "metastore_config" { + for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.metastore_config == null ? [] : [""] + content { + dataproc_metastore_service = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.metastore_config.dataproc_metastore_service + } + } + dynamic "spark_history_server_config" { + for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.spark_history_server_config == null ? [] : [""] + content { + dataproc_cluster = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.spark_history_server_config.dataproc_cluster + } + } + } + } + dynamic "kubernetes_cluster_config" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config == null ? [] : [""] + content { + kubernetes_namespace = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_namespace + dynamic "kubernetes_software_config" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config == null ? [] : [""] + content { + component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.component_version + properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.properties + } + } + + dynamic "gke_cluster_config" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config == null ? [] : [""] + content { + gke_cluster_target = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.gke_cluster_target + dynamic "node_pool_target" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target == null ? [] : [""] + content { + node_pool = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target.node_pool + roles = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target.roles + dynamic "node_pool_config" { + for_each = try(var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config == null ? [] : [""], []) + content { + dynamic "autoscaling" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling == null ? [] : [""] + content { + min_node_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling.min_node_count + max_node_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling.max_node_count + } + } + dynamic "config" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config == null ? [] : [""] + content { + machine_type = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.machine_type + local_ssd_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.local_ssd_count + preemptible = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.preemptible + min_cpu_platform = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.min_cpu_platform + spot = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.spot + } + } + locations = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.locations + } + } + } + } + } + } + } + } + } + } + lifecycle { + ignore_changes = [ + # Some scopes are assigned in addition to the one configured + # https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#dataproc_vm_access_scopes + cluster_config[0].gce_cluster_config[0].service_account_scopes, + ] + } +} diff --git a/modules/dataproc/output.tf b/modules/dataproc/output.tf new file mode 100644 index 00000000..797637d2 --- /dev/null +++ b/modules/dataproc/output.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Cloud Dataproc module output. + +output "bucket_names" { + description = "List of bucket names which have been assigned to the cluster." + value = google_dataproc_cluster.cluster.cluster_config.0.bucket +} + +output "name" { + description = "The name of the cluster." + value = google_dataproc_cluster.cluster.cluster_config.0.bucket +} + +output "http_ports" { + description = "The map of port descriptions to URLs." + value = google_dataproc_cluster.cluster.cluster_config.0.endpoint_config.0.http_ports +} + +output "instance_names" { + description = "List of instance names which have been assigned to the cluster." + value = { + master = google_dataproc_cluster.cluster.cluster_config.0.master_config.0.instance_names + worker = google_dataproc_cluster.cluster.cluster_config.0.worker_config.0.instance_names + preemptible_worker = google_dataproc_cluster.cluster.cluster_config.0.preemptible_worker_config.0.instance_names + } +} + diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf new file mode 100644 index 00000000..3636a706 --- /dev/null +++ b/modules/dataproc/variables.tf @@ -0,0 +1,240 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "dataproc_config" { + description = "Dataproc cluster config." + type = object({ + graceful_decommission_timeout = optional(string, null) + cluster_config = optional(object({ + staging_bucket = optional(string, null) + temp_bucket = optional(string, null) + gce_cluster_config = optional(object({ + zone = optional(string, null) + network = optional(string, null) + subnetwork = optional(string, null) + service_account = optional(string, null) + service_account_scopes = optional(list(string), null) + tags = optional(list(string), []) + internal_ip_only = optional(bool, null) + metadata = optional(map(string), {}) + reservation_affinity = optional(object({ + consume_reservation_type = string + key = string + values = string + }), null) + node_group_affinity = optional(object({ + node_group_uri = string + }), null) + + shielded_instance_config = optional(object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }), null) + }), null) + master_config = optional(object({ + num_instances = number + machine_type = string + min_cpu_platform = string + disk_config = optional(object({ + boot_disk_type = string + boot_disk_size_gb = number + num_local_ssds = number + }), null) + accelerators = optional(object({ + accelerator_type = string + accelerator_count = number + }), null) + }), null) + worker_config = optional(object({ + num_instances = number + machine_type = string + min_cpu_platform = string + disk_config = optional(object({ + boot_disk_type = string + boot_disk_size_gb = number + num_local_ssds = number + }), null) + image_uri = string + accelerators = optional(object({ + accelerator_type = string + accelerator_count = number + }), null) + }), null) + preemptible_worker_config = optional(object({ + num_instances = number + preemptibility = string + disk_config = optional(object({ + boot_disk_type = string + boot_disk_size_gb = number + num_local_ssds = number + }), null) + }), null) + software_config = optional(object({ + image_version = string + override_properties = list(map(string)) + optional_components = list(string) + }), null) + security_config = optional(object({ + kerberos_config = object({ + cross_realm_trust_admin_server = optional(string, null) + cross_realm_trust_kdc = optional(string, null) + cross_realm_trust_realm = optional(string, null) + cross_realm_trust_shared_password_uri = optional(string, null) + enable_kerberos = optional(string, null) + kdc_db_key_uri = optional(string, null) + key_password_uri = optional(string, null) + keystore_uri = optional(string, null) + keystore_password_uri = optional(string, null) + kms_key_uri = string + realm = optional(string, null) + root_principal_password_uri = string + tgt_lifetime_hours = optional(string, null) + truststore_password_uri = optional(string, null) + truststore_uri = optional(string, null) + }) + }), null) + autoscaling_config = optional(object({ + policy_uri = string + }), null) + initialization_action = optional(object({ + script = string + timeout_sec = optional(string, null) + }), null) + encryption_config = optional(object({ + kms_key_name = string + }), null) + lifecycle_config = optional(object({ + idle_delete_ttl = optional(string, null) + auto_delete_time = optional(string, null) + }), null) + endpoint_config = optional(object({ + enable_http_port_access = string + }), null) + dataproc_metric_config = optional(object({ + metrics = list(object({ + metric_source = string + metric_overrides = optional(string, null) + })) + }), null) + metastore_config = optional(object({ + dataproc_metastore_service = string + }), null) + }), null) + + virtual_cluster_config = optional(object({ + staging_bucket = optional(string, null) + auxiliary_services_config = optional(object({ + metastore_config = optional(object({ + dataproc_metastore_service = string + }), null) + spark_history_server_config = optional(object({ + dataproc_cluster = string + }), null) + }), null) + kubernetes_cluster_config = object({ + kubernetes_namespace = optional(string, null) + kubernetes_software_config = object({ + component_version = list(map(string)) + properties = optional(list(map(string)), null) + }) + + gke_cluster_config = object({ + gke_cluster_target = optional(string, null) + node_pool_target = optional(object({ + node_pool = string + roles = list(string) + node_pool_config = optional(object({ + autoscaling = optional(object({ + min_node_count = optional(number, null) + max_node_count = optional(number, null) + }), null) + + config = object({ + machine_type = optional(string, null) + preemptible = optional(bool, null) + local_ssd_count = optional(number, null) + min_cpu_platform = optional(string, null) + spot = optional(bool, null) + }) + + locations = optional(list(string), null) + }), null) + }), null) + }) + }) + }), null) + }) + default = {} +} + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_additive" { + description = "IAM additive bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "labels" { + description = "The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs." + type = map(string) + default = {} +} + +variable "name" { + description = "Cluster name." + type = string +} + +variable "prefix" { + description = "Optional prefix used to generate project id and name." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "Project ID." + type = string +} + +variable "region" { + description = "Dataproc region." + type = string +} + +variable "service_account" { + description = "Service account to set on the Dataproc cluster." + type = string + default = null +} diff --git a/modules/dataproc/versions.tf b/modules/dataproc/versions.tf new file mode 100644 index 00000000..08492c6f --- /dev/null +++ b/modules/dataproc/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3.1" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.50.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.50.0" # tftest + } + } +} + +