Merge pull request #1190 from GoogleCloudPlatform/lcaggio/dataproc
Dataproc Module
This commit is contained in:
commit
965375c78b
File diff suppressed because one or more lines are too long
|
@ -0,0 +1,67 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
# # tfdoc:file:description Generic IAM bindings and roles.
|
||||||
|
|
||||||
|
locals {
|
||||||
|
_group_iam_roles = distinct(flatten(values(var.group_iam)))
|
||||||
|
_group_iam = {
|
||||||
|
for r in local._group_iam_roles : r => [
|
||||||
|
for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null
|
||||||
|
]
|
||||||
|
}
|
||||||
|
_iam_additive_pairs = flatten([
|
||||||
|
for role, members in var.iam_additive : [
|
||||||
|
for member in members : { role = role, member = member }
|
||||||
|
]
|
||||||
|
])
|
||||||
|
iam = {
|
||||||
|
for role in distinct(concat(keys(var.iam), keys(local._group_iam))) :
|
||||||
|
role => concat(
|
||||||
|
try(var.iam[role], []),
|
||||||
|
try(local._group_iam[role], [])
|
||||||
|
)
|
||||||
|
}
|
||||||
|
iam_additive = {
|
||||||
|
for pair in local._iam_additive_pairs :
|
||||||
|
"${pair.role}-${pair.member}" => {
|
||||||
|
role = pair.role
|
||||||
|
member = pair.member
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "google_dataproc_cluster_iam_binding" "authoritative" {
|
||||||
|
for_each = local.iam
|
||||||
|
project = var.project_id
|
||||||
|
cluster = google_dataproc_cluster.cluster.name
|
||||||
|
region = var.region
|
||||||
|
role = each.key
|
||||||
|
members = each.value
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "google_dataproc_cluster_iam_member" "additive" {
|
||||||
|
for_each = (
|
||||||
|
length(var.iam_additive) > 0
|
||||||
|
? local.iam_additive
|
||||||
|
: {}
|
||||||
|
)
|
||||||
|
project = var.project_id
|
||||||
|
cluster = google_dataproc_cluster.cluster.name
|
||||||
|
region = var.region
|
||||||
|
role = each.value.role
|
||||||
|
member = each.value.member
|
||||||
|
}
|
|
@ -0,0 +1,298 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
# tfdoc:file:description Cloud Dataproc resource definition.
|
||||||
|
|
||||||
|
locals {
|
||||||
|
prefix = var.prefix == null ? "" : "${var.prefix}-"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "google_dataproc_cluster" "cluster" {
|
||||||
|
name = "${local.prefix}${var.name}"
|
||||||
|
project = var.project_id
|
||||||
|
region = var.region
|
||||||
|
graceful_decommission_timeout = var.dataproc_config.graceful_decommission_timeout
|
||||||
|
labels = var.labels
|
||||||
|
dynamic "cluster_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
staging_bucket = var.dataproc_config.cluster_config.staging_bucket
|
||||||
|
temp_bucket = var.dataproc_config.cluster_config.temp_bucket
|
||||||
|
dynamic "gce_cluster_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.gce_cluster_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
zone = var.dataproc_config.cluster_config.gce_cluster_config.zone
|
||||||
|
network = var.dataproc_config.cluster_config.gce_cluster_config.network
|
||||||
|
subnetwork = var.dataproc_config.cluster_config.gce_cluster_config.subnetwork
|
||||||
|
service_account = var.dataproc_config.cluster_config.gce_cluster_config.service_account
|
||||||
|
service_account_scopes = var.dataproc_config.cluster_config.gce_cluster_config.service_account_scopes
|
||||||
|
tags = var.dataproc_config.cluster_config.gce_cluster_config.tags
|
||||||
|
internal_ip_only = var.dataproc_config.cluster_config.gce_cluster_config.internal_ip_only
|
||||||
|
metadata = var.dataproc_config.cluster_config.gce_cluster_config.metadata
|
||||||
|
dynamic "reservation_affinity" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
consume_reservation_type = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.consume_reservation_type
|
||||||
|
key = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.key
|
||||||
|
values = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "node_group_affinity" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.gce_cluster_config.node_group_affinity == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
node_group_uri = var.dataproc_config.cluster_config.gce_cluster_config.node_group_uri
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "shielded_instance_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_secure_boot
|
||||||
|
enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_vtpm
|
||||||
|
enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_integrity_monitoring
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "master_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.master_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
num_instances = var.dataproc_config.cluster_config.master_config.num_instances
|
||||||
|
machine_type = var.dataproc_config.cluster_config.master_config.machine_type
|
||||||
|
min_cpu_platform = var.dataproc_config.cluster_config.master_config.min_cpu_platform
|
||||||
|
image_uri = var.dataproc_config.cluster_config.master_config.image_uri
|
||||||
|
dynamic "disk_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.master_config.disk_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
boot_disk_type = var.dataproc_config.cluster_config.master_config.disk_config.boot_disk_type
|
||||||
|
boot_disk_size_gb = var.dataproc_config.cluster_config.master_config.disk_config.boot_disk_size_gb
|
||||||
|
num_local_ssds = var.dataproc_config.cluster_config.master_config.disk_config.num_local_ssds
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "accelerators" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.master_config.accelerators == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
accelerator_type = var.dataproc_config.cluster_config.master_config.accelerators.accelerator_type
|
||||||
|
accelerator_count = var.dataproc_config.cluster_config.master_config.accelerators.accelerator_count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "worker_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.worker_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
num_instances = var.dataproc_config.cluster_config.worker_config.num_instances
|
||||||
|
machine_type = var.dataproc_config.cluster_config.worker_config.machine_type
|
||||||
|
min_cpu_platform = var.dataproc_config.cluster_config.worker_config.min_cpu_platform
|
||||||
|
dynamic "disk_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.worker_config.disk_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_type
|
||||||
|
boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_size_gb
|
||||||
|
num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.value.num_local_ssds
|
||||||
|
}
|
||||||
|
}
|
||||||
|
image_uri = var.dataproc_config.cluster_config.worker_config.image_uri
|
||||||
|
dynamic "accelerators" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.worker_config.accelerators == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
accelerator_type = var.dataproc_config.cluster_config.accelerators.accelerator_type
|
||||||
|
accelerator_count = var.dataproc_config.cluster_config.accelerators.accelerator_count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "preemptible_worker_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.preemptible_worker_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
num_instances = var.dataproc_config.cluster_config.preemptible_worker_config.num_instances
|
||||||
|
preemptibility = var.dataproc_config.cluster_config.preemptible_worker_config.preemptibility
|
||||||
|
dynamic "disk_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.preemptible_worker_config.disk_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
boot_disk_type = var.dataproc_config.cluster_config.disk_config.boot_disk_type
|
||||||
|
boot_disk_size_gb = var.dataproc_config.cluster_config.disk_config.boot_disk_size_gb
|
||||||
|
num_local_ssds = var.dataproc_config.cluster_config.disk_config.num_local_ssds
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "software_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.software_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
image_version = var.dataproc_config.cluster_config.software_config.image_version
|
||||||
|
override_properties = var.dataproc_config.cluster_config.software_config.override_properties
|
||||||
|
optional_components = var.dataproc_config.cluster_config.software_config.optional_components
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "security_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.security_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
dynamic "kerberos_config" {
|
||||||
|
for_each = try(var.dataproc_config.cluster_config.security_config.kerberos_config == null ? [] : [""], [])
|
||||||
|
content {
|
||||||
|
cross_realm_trust_admin_server = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_admin_server
|
||||||
|
cross_realm_trust_kdc = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_kdc
|
||||||
|
cross_realm_trust_realm = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_realm
|
||||||
|
cross_realm_trust_shared_password_uri = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_shared_password_uri
|
||||||
|
enable_kerberos = var.dataproc_config.cluster_config.kerberos_config.enable_kerberos
|
||||||
|
kdc_db_key_uri = var.dataproc_config.cluster_config.kerberos_config.kdc_db_key_uri
|
||||||
|
key_password_uri = var.dataproc_config.cluster_config.kerberos_config.key_password_uri
|
||||||
|
keystore_uri = var.dataproc_config.cluster_config.kerberos_config.keystore_uri
|
||||||
|
keystore_password_uri = var.dataproc_config.cluster_config.kerberos_config.keystore_password_uri
|
||||||
|
kms_key_uri = var.dataproc_config.cluster_config.kerberos_config.kms_key_uri
|
||||||
|
realm = var.dataproc_config.cluster_config.kerberos_config.realm
|
||||||
|
root_principal_password_uri = var.dataproc_config.cluster_config.kerberos_config.root_principal_password_uri
|
||||||
|
tgt_lifetime_hours = var.dataproc_config.cluster_config.kerberos_config.tgt_lifetime_hours
|
||||||
|
truststore_password_uri = var.dataproc_config.cluster_config.kerberos_config.truststore_password_uri
|
||||||
|
truststore_uri = var.dataproc_config.cluster_config.kerberos_config.truststore_uri
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "autoscaling_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.autoscaling_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
policy_uri = var.dataproc_config.cluster_config.autoscaling_config.value.policy_uri
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "initialization_action" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.initialization_action == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
script = var.dataproc_config.cluster_config.initialization_action.value.script
|
||||||
|
timeout_sec = var.dataproc_config.cluster_config.initialization_action.value.timeout_sec
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "encryption_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.encryption_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
kms_key_name = var.dataproc_config.cluster_config.encryption_config.value.kms_key_name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "dataproc_metric_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.dataproc_metric_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
dynamic "metrics" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.dataproc_metric_config.metrics == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
metric_source = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_source
|
||||||
|
metric_overrides = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_overrides
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "lifecycle_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.lifecycle_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
idle_delete_ttl = var.dataproc_config.cluster_config.lifecycle_config.idle_delete_ttl
|
||||||
|
auto_delete_time = var.dataproc_config.cluster_config.lifecycle_config.auto_delete_time
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "endpoint_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.endpoint_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
enable_http_port_access = var.dataproc_config.cluster_config.endpoint_config.enable_http_port_access
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "metastore_config" {
|
||||||
|
for_each = var.dataproc_config.cluster_config.metastore_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
dataproc_metastore_service = var.dataproc_config.cluster_config.metastore_config.dataproc_metastore_service
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dynamic "virtual_cluster_config" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
dynamic "auxiliary_services_config" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
dynamic "metastore_config" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.metastore_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
dataproc_metastore_service = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.metastore_config.dataproc_metastore_service
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "spark_history_server_config" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.spark_history_server_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
dataproc_cluster = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.spark_history_server_config.dataproc_cluster
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "kubernetes_cluster_config" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
kubernetes_namespace = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_namespace
|
||||||
|
dynamic "kubernetes_software_config" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.component_version
|
||||||
|
properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.properties
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dynamic "gke_cluster_config" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
gke_cluster_target = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.gke_cluster_target
|
||||||
|
dynamic "node_pool_target" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
node_pool = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target.node_pool
|
||||||
|
roles = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target.roles
|
||||||
|
dynamic "node_pool_config" {
|
||||||
|
for_each = try(var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config == null ? [] : [""], [])
|
||||||
|
content {
|
||||||
|
dynamic "autoscaling" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
min_node_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling.min_node_count
|
||||||
|
max_node_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling.max_node_count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dynamic "config" {
|
||||||
|
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config == null ? [] : [""]
|
||||||
|
content {
|
||||||
|
machine_type = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.machine_type
|
||||||
|
local_ssd_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.local_ssd_count
|
||||||
|
preemptible = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.preemptible
|
||||||
|
min_cpu_platform = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.min_cpu_platform
|
||||||
|
spot = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.spot
|
||||||
|
}
|
||||||
|
}
|
||||||
|
locations = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.locations
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lifecycle {
|
||||||
|
ignore_changes = [
|
||||||
|
# Some scopes are assigned in addition to the one configured
|
||||||
|
# https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#dataproc_vm_access_scopes
|
||||||
|
cluster_config[0].gce_cluster_config[0].service_account_scopes,
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,42 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2022 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
# tfdoc:file:description Cloud Dataproc module output.
|
||||||
|
|
||||||
|
output "bucket_names" {
|
||||||
|
description = "List of bucket names which have been assigned to the cluster."
|
||||||
|
value = google_dataproc_cluster.cluster.cluster_config.0.bucket
|
||||||
|
}
|
||||||
|
|
||||||
|
output "name" {
|
||||||
|
description = "The name of the cluster."
|
||||||
|
value = google_dataproc_cluster.cluster.cluster_config.0.bucket
|
||||||
|
}
|
||||||
|
|
||||||
|
output "http_ports" {
|
||||||
|
description = "The map of port descriptions to URLs."
|
||||||
|
value = google_dataproc_cluster.cluster.cluster_config.0.endpoint_config.0.http_ports
|
||||||
|
}
|
||||||
|
|
||||||
|
output "instance_names" {
|
||||||
|
description = "List of instance names which have been assigned to the cluster."
|
||||||
|
value = {
|
||||||
|
master = google_dataproc_cluster.cluster.cluster_config.0.master_config.0.instance_names
|
||||||
|
worker = google_dataproc_cluster.cluster.cluster_config.0.worker_config.0.instance_names
|
||||||
|
preemptible_worker = google_dataproc_cluster.cluster.cluster_config.0.preemptible_worker_config.0.instance_names
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,240 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2023 Google LLC
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
variable "dataproc_config" {
|
||||||
|
description = "Dataproc cluster config."
|
||||||
|
type = object({
|
||||||
|
graceful_decommission_timeout = optional(string, null)
|
||||||
|
cluster_config = optional(object({
|
||||||
|
staging_bucket = optional(string, null)
|
||||||
|
temp_bucket = optional(string, null)
|
||||||
|
gce_cluster_config = optional(object({
|
||||||
|
zone = optional(string, null)
|
||||||
|
network = optional(string, null)
|
||||||
|
subnetwork = optional(string, null)
|
||||||
|
service_account = optional(string, null)
|
||||||
|
service_account_scopes = optional(list(string), null)
|
||||||
|
tags = optional(list(string), [])
|
||||||
|
internal_ip_only = optional(bool, null)
|
||||||
|
metadata = optional(map(string), {})
|
||||||
|
reservation_affinity = optional(object({
|
||||||
|
consume_reservation_type = string
|
||||||
|
key = string
|
||||||
|
values = string
|
||||||
|
}), null)
|
||||||
|
node_group_affinity = optional(object({
|
||||||
|
node_group_uri = string
|
||||||
|
}), null)
|
||||||
|
|
||||||
|
shielded_instance_config = optional(object({
|
||||||
|
enable_secure_boot = bool
|
||||||
|
enable_vtpm = bool
|
||||||
|
enable_integrity_monitoring = bool
|
||||||
|
}), null)
|
||||||
|
}), null)
|
||||||
|
master_config = optional(object({
|
||||||
|
num_instances = number
|
||||||
|
machine_type = string
|
||||||
|
min_cpu_platform = string
|
||||||
|
disk_config = optional(object({
|
||||||
|
boot_disk_type = string
|
||||||
|
boot_disk_size_gb = number
|
||||||
|
num_local_ssds = number
|
||||||
|
}), null)
|
||||||
|
accelerators = optional(object({
|
||||||
|
accelerator_type = string
|
||||||
|
accelerator_count = number
|
||||||
|
}), null)
|
||||||
|
}), null)
|
||||||
|
worker_config = optional(object({
|
||||||
|
num_instances = number
|
||||||
|
machine_type = string
|
||||||
|
min_cpu_platform = string
|
||||||
|
disk_config = optional(object({
|
||||||
|
boot_disk_type = string
|
||||||
|
boot_disk_size_gb = number
|
||||||
|
num_local_ssds = number
|
||||||
|
}), null)
|
||||||
|
image_uri = string
|
||||||
|
accelerators = optional(object({
|
||||||
|
accelerator_type = string
|
||||||
|
accelerator_count = number
|
||||||
|
}), null)
|
||||||
|
}), null)
|
||||||
|
preemptible_worker_config = optional(object({
|
||||||
|
num_instances = number
|
||||||
|
preemptibility = string
|
||||||
|
disk_config = optional(object({
|
||||||
|
boot_disk_type = string
|
||||||
|
boot_disk_size_gb = number
|
||||||
|
num_local_ssds = number
|
||||||
|
}), null)
|
||||||
|
}), null)
|
||||||
|
software_config = optional(object({
|
||||||
|
image_version = string
|
||||||
|
override_properties = list(map(string))
|
||||||
|
optional_components = list(string)
|
||||||
|
}), null)
|
||||||
|
security_config = optional(object({
|
||||||
|
kerberos_config = object({
|
||||||
|
cross_realm_trust_admin_server = optional(string, null)
|
||||||
|
cross_realm_trust_kdc = optional(string, null)
|
||||||
|
cross_realm_trust_realm = optional(string, null)
|
||||||
|
cross_realm_trust_shared_password_uri = optional(string, null)
|
||||||
|
enable_kerberos = optional(string, null)
|
||||||
|
kdc_db_key_uri = optional(string, null)
|
||||||
|
key_password_uri = optional(string, null)
|
||||||
|
keystore_uri = optional(string, null)
|
||||||
|
keystore_password_uri = optional(string, null)
|
||||||
|
kms_key_uri = string
|
||||||
|
realm = optional(string, null)
|
||||||
|
root_principal_password_uri = string
|
||||||
|
tgt_lifetime_hours = optional(string, null)
|
||||||
|
truststore_password_uri = optional(string, null)
|
||||||
|
truststore_uri = optional(string, null)
|
||||||
|
})
|
||||||
|
}), null)
|
||||||
|
autoscaling_config = optional(object({
|
||||||
|
policy_uri = string
|
||||||
|
}), null)
|
||||||
|
initialization_action = optional(object({
|
||||||
|
script = string
|
||||||
|
timeout_sec = optional(string, null)
|
||||||
|
}), null)
|
||||||
|
encryption_config = optional(object({
|
||||||
|
kms_key_name = string
|
||||||
|
}), null)
|
||||||
|
lifecycle_config = optional(object({
|
||||||
|
idle_delete_ttl = optional(string, null)
|
||||||
|
auto_delete_time = optional(string, null)
|
||||||
|
}), null)
|
||||||
|
endpoint_config = optional(object({
|
||||||
|
enable_http_port_access = string
|
||||||
|
}), null)
|
||||||
|
dataproc_metric_config = optional(object({
|
||||||
|
metrics = list(object({
|
||||||
|
metric_source = string
|
||||||
|
metric_overrides = optional(string, null)
|
||||||
|
}))
|
||||||
|
}), null)
|
||||||
|
metastore_config = optional(object({
|
||||||
|
dataproc_metastore_service = string
|
||||||
|
}), null)
|
||||||
|
}), null)
|
||||||
|
|
||||||
|
virtual_cluster_config = optional(object({
|
||||||
|
staging_bucket = optional(string, null)
|
||||||
|
auxiliary_services_config = optional(object({
|
||||||
|
metastore_config = optional(object({
|
||||||
|
dataproc_metastore_service = string
|
||||||
|
}), null)
|
||||||
|
spark_history_server_config = optional(object({
|
||||||
|
dataproc_cluster = string
|
||||||
|
}), null)
|
||||||
|
}), null)
|
||||||
|
kubernetes_cluster_config = object({
|
||||||
|
kubernetes_namespace = optional(string, null)
|
||||||
|
kubernetes_software_config = object({
|
||||||
|
component_version = list(map(string))
|
||||||
|
properties = optional(list(map(string)), null)
|
||||||
|
})
|
||||||
|
|
||||||
|
gke_cluster_config = object({
|
||||||
|
gke_cluster_target = optional(string, null)
|
||||||
|
node_pool_target = optional(object({
|
||||||
|
node_pool = string
|
||||||
|
roles = list(string)
|
||||||
|
node_pool_config = optional(object({
|
||||||
|
autoscaling = optional(object({
|
||||||
|
min_node_count = optional(number, null)
|
||||||
|
max_node_count = optional(number, null)
|
||||||
|
}), null)
|
||||||
|
|
||||||
|
config = object({
|
||||||
|
machine_type = optional(string, null)
|
||||||
|
preemptible = optional(bool, null)
|
||||||
|
local_ssd_count = optional(number, null)
|
||||||
|
min_cpu_platform = optional(string, null)
|
||||||
|
spot = optional(bool, null)
|
||||||
|
})
|
||||||
|
|
||||||
|
locations = optional(list(string), null)
|
||||||
|
}), null)
|
||||||
|
}), null)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}), null)
|
||||||
|
})
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "group_iam" {
|
||||||
|
description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable."
|
||||||
|
type = map(list(string))
|
||||||
|
default = {}
|
||||||
|
nullable = false
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "iam" {
|
||||||
|
description = "IAM bindings in {ROLE => [MEMBERS]} format."
|
||||||
|
type = map(list(string))
|
||||||
|
default = {}
|
||||||
|
nullable = false
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "iam_additive" {
|
||||||
|
description = "IAM additive bindings in {ROLE => [MEMBERS]} format."
|
||||||
|
type = map(list(string))
|
||||||
|
default = {}
|
||||||
|
nullable = false
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "labels" {
|
||||||
|
description = "The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs."
|
||||||
|
type = map(string)
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "name" {
|
||||||
|
description = "Cluster name."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "prefix" {
|
||||||
|
description = "Optional prefix used to generate project id and name."
|
||||||
|
type = string
|
||||||
|
default = null
|
||||||
|
validation {
|
||||||
|
condition = var.prefix != ""
|
||||||
|
error_message = "Prefix cannot be empty, please use null instead."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "project_id" {
|
||||||
|
description = "Project ID."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "region" {
|
||||||
|
description = "Dataproc region."
|
||||||
|
type = string
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "service_account" {
|
||||||
|
description = "Service account to set on the Dataproc cluster."
|
||||||
|
type = string
|
||||||
|
default = null
|
||||||
|
}
|
|
@ -0,0 +1,29 @@
|
||||||
|
# Copyright 2022 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
terraform {
|
||||||
|
required_version = ">= 1.3.1"
|
||||||
|
required_providers {
|
||||||
|
google = {
|
||||||
|
source = "hashicorp/google"
|
||||||
|
version = ">= 4.50.0" # tftest
|
||||||
|
}
|
||||||
|
google-beta = {
|
||||||
|
source = "hashicorp/google-beta"
|
||||||
|
version = ">= 4.50.0" # tftest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue