Merge pull request #1190 from GoogleCloudPlatform/lcaggio/dataproc
Dataproc Module
This commit is contained in:
lcaggio
GitHub
commit
965375c78b
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
# # tfdoc:file:description Generic IAM bindings and roles.
|
||||
|
||||
locals {
|
||||
_group_iam_roles = distinct(flatten(values(var.group_iam)))
|
||||
_group_iam = {
|
||||
for r in local._group_iam_roles : r => [
|
||||
for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null
|
||||
]
|
||||
}
|
||||
_iam_additive_pairs = flatten([
|
||||
for role, members in var.iam_additive : [
|
||||
for member in members : { role = role, member = member }
|
||||
]
|
||||
])
|
||||
iam = {
|
||||
for role in distinct(concat(keys(var.iam), keys(local._group_iam))) :
|
||||
role => concat(
|
||||
try(var.iam[role], []),
|
||||
try(local._group_iam[role], [])
|
||||
)
|
||||
}
|
||||
iam_additive = {
|
||||
for pair in local._iam_additive_pairs :
|
||||
"${pair.role}-${pair.member}" => {
|
||||
role = pair.role
|
||||
member = pair.member
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_dataproc_cluster_iam_binding" "authoritative" {
|
||||
for_each = local.iam
|
||||
project = var.project_id
|
||||
cluster = google_dataproc_cluster.cluster.name
|
||||
region = var.region
|
||||
role = each.key
|
||||
members = each.value
|
||||
}
|
||||
|
||||
resource "google_dataproc_cluster_iam_member" "additive" {
|
||||
for_each = (
|
||||
length(var.iam_additive) > 0
|
||||
? local.iam_additive
|
||||
: {}
|
||||
)
|
||||
project = var.project_id
|
||||
cluster = google_dataproc_cluster.cluster.name
|
||||
region = var.region
|
||||
role = each.value.role
|
||||
member = each.value.member
|
||||
}
|
||||
|
|
@ -0,0 +1,298 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
# tfdoc:file:description Cloud Dataproc resource definition.
|
||||
|
||||
locals {
|
||||
prefix = var.prefix == null ? "" : "${var.prefix}-"
|
||||
}
|
||||
|
||||
resource "google_dataproc_cluster" "cluster" {
|
||||
name = "${local.prefix}${var.name}"
|
||||
project = var.project_id
|
||||
region = var.region
|
||||
graceful_decommission_timeout = var.dataproc_config.graceful_decommission_timeout
|
||||
labels = var.labels
|
||||
dynamic "cluster_config" {
|
||||
for_each = var.dataproc_config.cluster_config == null ? [] : [""]
|
||||
content {
|
||||
staging_bucket = var.dataproc_config.cluster_config.staging_bucket
|
||||
temp_bucket = var.dataproc_config.cluster_config.temp_bucket
|
||||
dynamic "gce_cluster_config" {
|
||||
for_each = var.dataproc_config.cluster_config.gce_cluster_config == null ? [] : [""]
|
||||
content {
|
||||
zone = var.dataproc_config.cluster_config.gce_cluster_config.zone
|
||||
network = var.dataproc_config.cluster_config.gce_cluster_config.network
|
||||
subnetwork = var.dataproc_config.cluster_config.gce_cluster_config.subnetwork
|
||||
service_account = var.dataproc_config.cluster_config.gce_cluster_config.service_account
|
||||
service_account_scopes = var.dataproc_config.cluster_config.gce_cluster_config.service_account_scopes
|
||||
tags = var.dataproc_config.cluster_config.gce_cluster_config.tags
|
||||
internal_ip_only = var.dataproc_config.cluster_config.gce_cluster_config.internal_ip_only
|
||||
metadata = var.dataproc_config.cluster_config.gce_cluster_config.metadata
|
||||
dynamic "reservation_affinity" {
|
||||
for_each = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity == null ? [] : [""]
|
||||
content {
|
||||
consume_reservation_type = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.consume_reservation_type
|
||||
key = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.key
|
||||
values = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.value
|
||||
}
|
||||
}
|
||||
dynamic "node_group_affinity" {
|
||||
for_each = var.dataproc_config.cluster_config.gce_cluster_config.node_group_affinity == null ? [] : [""]
|
||||
content {
|
||||
node_group_uri = var.dataproc_config.cluster_config.gce_cluster_config.node_group_uri
|
||||
}
|
||||
}
|
||||
dynamic "shielded_instance_config" {
|
||||
for_each = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config == null ? [] : [""]
|
||||
content {
|
||||
enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_secure_boot
|
||||
enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_vtpm
|
||||
enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_integrity_monitoring
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dynamic "master_config" {
|
||||
for_each = var.dataproc_config.cluster_config.master_config == null ? [] : [""]
|
||||
content {
|
||||
num_instances = var.dataproc_config.cluster_config.master_config.num_instances
|
||||
machine_type = var.dataproc_config.cluster_config.master_config.machine_type
|
||||
min_cpu_platform = var.dataproc_config.cluster_config.master_config.min_cpu_platform
|
||||
image_uri = var.dataproc_config.cluster_config.master_config.image_uri
|
||||
dynamic "disk_config" {
|
||||
for_each = var.dataproc_config.cluster_config.master_config.disk_config == null ? [] : [""]
|
||||
content {
|
||||
boot_disk_type = var.dataproc_config.cluster_config.master_config.disk_config.boot_disk_type
|
||||
boot_disk_size_gb = var.dataproc_config.cluster_config.master_config.disk_config.boot_disk_size_gb
|
||||
num_local_ssds = var.dataproc_config.cluster_config.master_config.disk_config.num_local_ssds
|
||||
}
|
||||
}
|
||||
dynamic "accelerators" {
|
||||
for_each = var.dataproc_config.cluster_config.master_config.accelerators == null ? [] : [""]
|
||||
content {
|
||||
accelerator_type = var.dataproc_config.cluster_config.master_config.accelerators.accelerator_type
|
||||
accelerator_count = var.dataproc_config.cluster_config.master_config.accelerators.accelerator_count
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dynamic "worker_config" {
|
||||
for_each = var.dataproc_config.cluster_config.worker_config == null ? [] : [""]
|
||||
content {
|
||||
num_instances = var.dataproc_config.cluster_config.worker_config.num_instances
|
||||
machine_type = var.dataproc_config.cluster_config.worker_config.machine_type
|
||||
min_cpu_platform = var.dataproc_config.cluster_config.worker_config.min_cpu_platform
|
||||
dynamic "disk_config" {
|
||||
for_each = var.dataproc_config.cluster_config.worker_config.disk_config == null ? [] : [""]
|
||||
content {
|
||||
boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_type
|
||||
boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_size_gb
|
||||
num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.value.num_local_ssds
|
||||
}
|
||||
}
|
||||
image_uri = var.dataproc_config.cluster_config.worker_config.image_uri
|
||||
dynamic "accelerators" {
|
||||
for_each = var.dataproc_config.cluster_config.worker_config.accelerators == null ? [] : [""]
|
||||
content {
|
||||
accelerator_type = var.dataproc_config.cluster_config.accelerators.accelerator_type
|
||||
accelerator_count = var.dataproc_config.cluster_config.accelerators.accelerator_count
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dynamic "preemptible_worker_config" {
|
||||
for_each = var.dataproc_config.cluster_config.preemptible_worker_config == null ? [] : [""]
|
||||
content {
|
||||
num_instances = var.dataproc_config.cluster_config.preemptible_worker_config.num_instances
|
||||
preemptibility = var.dataproc_config.cluster_config.preemptible_worker_config.preemptibility
|
||||
dynamic "disk_config" {
|
||||
for_each = var.dataproc_config.cluster_config.preemptible_worker_config.disk_config == null ? [] : [""]
|
||||
content {
|
||||
boot_disk_type = var.dataproc_config.cluster_config.disk_config.boot_disk_type
|
||||
boot_disk_size_gb = var.dataproc_config.cluster_config.disk_config.boot_disk_size_gb
|
||||
num_local_ssds = var.dataproc_config.cluster_config.disk_config.num_local_ssds
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dynamic "software_config" {
|
||||
for_each = var.dataproc_config.cluster_config.software_config == null ? [] : [""]
|
||||
content {
|
||||
image_version = var.dataproc_config.cluster_config.software_config.image_version
|
||||
override_properties = var.dataproc_config.cluster_config.software_config.override_properties
|
||||
optional_components = var.dataproc_config.cluster_config.software_config.optional_components
|
||||
}
|
||||
}
|
||||
dynamic "security_config" {
|
||||
for_each = var.dataproc_config.cluster_config.security_config == null ? [] : [""]
|
||||
content {
|
||||
dynamic "kerberos_config" {
|
||||
for_each = try(var.dataproc_config.cluster_config.security_config.kerberos_config == null ? [] : [""], [])
|
||||
content {
|
||||
cross_realm_trust_admin_server = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_admin_server
|
||||
cross_realm_trust_kdc = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_kdc
|
||||
cross_realm_trust_realm = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_realm
|
||||
cross_realm_trust_shared_password_uri = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_shared_password_uri
|
||||
enable_kerberos = var.dataproc_config.cluster_config.kerberos_config.enable_kerberos
|
||||
kdc_db_key_uri = var.dataproc_config.cluster_config.kerberos_config.kdc_db_key_uri
|
||||
key_password_uri = var.dataproc_config.cluster_config.kerberos_config.key_password_uri
|
||||
keystore_uri = var.dataproc_config.cluster_config.kerberos_config.keystore_uri
|
||||
keystore_password_uri = var.dataproc_config.cluster_config.kerberos_config.keystore_password_uri
|
||||
kms_key_uri = var.dataproc_config.cluster_config.kerberos_config.kms_key_uri
|
||||
realm = var.dataproc_config.cluster_config.kerberos_config.realm
|
||||
root_principal_password_uri = var.dataproc_config.cluster_config.kerberos_config.root_principal_password_uri
|
||||
tgt_lifetime_hours = var.dataproc_config.cluster_config.kerberos_config.tgt_lifetime_hours
|
||||
truststore_password_uri = var.dataproc_config.cluster_config.kerberos_config.truststore_password_uri
|
||||
truststore_uri = var.dataproc_config.cluster_config.kerberos_config.truststore_uri
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dynamic "autoscaling_config" {
|
||||
for_each = var.dataproc_config.cluster_config.autoscaling_config == null ? [] : [""]
|
||||
content {
|
||||
policy_uri = var.dataproc_config.cluster_config.autoscaling_config.value.policy_uri
|
||||
}
|
||||
}
|
||||
dynamic "initialization_action" {
|
||||
for_each = var.dataproc_config.cluster_config.initialization_action == null ? [] : [""]
|
||||
content {
|
||||
script = var.dataproc_config.cluster_config.initialization_action.value.script
|
||||
timeout_sec = var.dataproc_config.cluster_config.initialization_action.value.timeout_sec
|
||||
}
|
||||
}
|
||||
dynamic "encryption_config" {
|
||||
for_each = var.dataproc_config.cluster_config.encryption_config == null ? [] : [""]
|
||||
content {
|
||||
kms_key_name = var.dataproc_config.cluster_config.encryption_config.value.kms_key_name
|
||||
}
|
||||
}
|
||||
dynamic "dataproc_metric_config" {
|
||||
for_each = var.dataproc_config.cluster_config.dataproc_metric_config == null ? [] : [""]
|
||||
content {
|
||||
dynamic "metrics" {
|
||||
for_each = var.dataproc_config.cluster_config.dataproc_metric_config.metrics == null ? [] : [""]
|
||||
content {
|
||||
metric_source = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_source
|
||||
metric_overrides = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_overrides
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dynamic "lifecycle_config" {
|
||||
for_each = var.dataproc_config.cluster_config.lifecycle_config == null ? [] : [""]
|
||||
content {
|
||||
idle_delete_ttl = var.dataproc_config.cluster_config.lifecycle_config.idle_delete_ttl
|
||||
auto_delete_time = var.dataproc_config.cluster_config.lifecycle_config.auto_delete_time
|
||||
}
|
||||
}
|
||||
dynamic "endpoint_config" {
|
||||
for_each = var.dataproc_config.cluster_config.endpoint_config == null ? [] : [""]
|
||||
content {
|
||||
enable_http_port_access = var.dataproc_config.cluster_config.endpoint_config.enable_http_port_access
|
||||
}
|
||||
}
|
||||
dynamic "metastore_config" {
|
||||
for_each = var.dataproc_config.cluster_config.metastore_config == null ? [] : [""]
|
||||
content {
|
||||
dataproc_metastore_service = var.dataproc_config.cluster_config.metastore_config.dataproc_metastore_service
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
dynamic "virtual_cluster_config" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config == null ? [] : [""]
|
||||
content {
|
||||
dynamic "auxiliary_services_config" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config == null ? [] : [""]
|
||||
content {
|
||||
dynamic "metastore_config" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.metastore_config == null ? [] : [""]
|
||||
content {
|
||||
dataproc_metastore_service = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.metastore_config.dataproc_metastore_service
|
||||
}
|
||||
}
|
||||
dynamic "spark_history_server_config" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.spark_history_server_config == null ? [] : [""]
|
||||
content {
|
||||
dataproc_cluster = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.spark_history_server_config.dataproc_cluster
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dynamic "kubernetes_cluster_config" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config == null ? [] : [""]
|
||||
content {
|
||||
kubernetes_namespace = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_namespace
|
||||
dynamic "kubernetes_software_config" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config == null ? [] : [""]
|
||||
content {
|
||||
component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.component_version
|
||||
properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.properties
|
||||
}
|
||||
}
|
||||
|
||||
dynamic "gke_cluster_config" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config == null ? [] : [""]
|
||||
content {
|
||||
gke_cluster_target = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.gke_cluster_target
|
||||
dynamic "node_pool_target" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target == null ? [] : [""]
|
||||
content {
|
||||
node_pool = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target.node_pool
|
||||
roles = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target.roles
|
||||
dynamic "node_pool_config" {
|
||||
for_each = try(var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config == null ? [] : [""], [])
|
||||
content {
|
||||
dynamic "autoscaling" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling == null ? [] : [""]
|
||||
content {
|
||||
min_node_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling.min_node_count
|
||||
max_node_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling.max_node_count
|
||||
}
|
||||
}
|
||||
dynamic "config" {
|
||||
for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config == null ? [] : [""]
|
||||
content {
|
||||
machine_type = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.machine_type
|
||||
local_ssd_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.local_ssd_count
|
||||
preemptible = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.preemptible
|
||||
min_cpu_platform = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.min_cpu_platform
|
||||
spot = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.spot
|
||||
}
|
||||
}
|
||||
locations = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.locations
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lifecycle {
|
||||
ignore_changes = [
|
||||
# Some scopes are assigned in addition to the one configured
|
||||
# https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#dataproc_vm_access_scopes
|
||||
cluster_config[0].gce_cluster_config[0].service_account_scopes,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
# tfdoc:file:description Cloud Dataproc module output.
|
||||
|
||||
output "bucket_names" {
|
||||
description = "List of bucket names which have been assigned to the cluster."
|
||||
value = google_dataproc_cluster.cluster.cluster_config.0.bucket
|
||||
}
|
||||
|
||||
output "name" {
|
||||
description = "The name of the cluster."
|
||||
value = google_dataproc_cluster.cluster.cluster_config.0.bucket
|
||||
}
|
||||
|
||||
output "http_ports" {
|
||||
description = "The map of port descriptions to URLs."
|
||||
value = google_dataproc_cluster.cluster.cluster_config.0.endpoint_config.0.http_ports
|
||||
}
|
||||
|
||||
output "instance_names" {
|
||||
description = "List of instance names which have been assigned to the cluster."
|
||||
value = {
|
||||
master = google_dataproc_cluster.cluster.cluster_config.0.master_config.0.instance_names
|
||||
worker = google_dataproc_cluster.cluster.cluster_config.0.worker_config.0.instance_names
|
||||
preemptible_worker = google_dataproc_cluster.cluster.cluster_config.0.preemptible_worker_config.0.instance_names
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,240 @@
|
|||
/**
|
||||
* Copyright 2023 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
variable "dataproc_config" {
|
||||
description = "Dataproc cluster config."
|
||||
type = object({
|
||||
graceful_decommission_timeout = optional(string, null)
|
||||
cluster_config = optional(object({
|
||||
staging_bucket = optional(string, null)
|
||||
temp_bucket = optional(string, null)
|
||||
gce_cluster_config = optional(object({
|
||||
zone = optional(string, null)
|
||||
network = optional(string, null)
|
||||
subnetwork = optional(string, null)
|
||||
service_account = optional(string, null)
|
||||
service_account_scopes = optional(list(string), null)
|
||||
tags = optional(list(string), [])
|
||||
internal_ip_only = optional(bool, null)
|
||||
metadata = optional(map(string), {})
|
||||
reservation_affinity = optional(object({
|
||||
consume_reservation_type = string
|
||||
key = string
|
||||
values = string
|
||||
}), null)
|
||||
node_group_affinity = optional(object({
|
||||
node_group_uri = string
|
||||
}), null)
|
||||
|
||||
shielded_instance_config = optional(object({
|
||||
enable_secure_boot = bool
|
||||
enable_vtpm = bool
|
||||
enable_integrity_monitoring = bool
|
||||
}), null)
|
||||
}), null)
|
||||
master_config = optional(object({
|
||||
num_instances = number
|
||||
machine_type = string
|
||||
min_cpu_platform = string
|
||||
disk_config = optional(object({
|
||||
boot_disk_type = string
|
||||
boot_disk_size_gb = number
|
||||
num_local_ssds = number
|
||||
}), null)
|
||||
accelerators = optional(object({
|
||||
accelerator_type = string
|
||||
accelerator_count = number
|
||||
}), null)
|
||||
}), null)
|
||||
worker_config = optional(object({
|
||||
num_instances = number
|
||||
machine_type = string
|
||||
min_cpu_platform = string
|
||||
disk_config = optional(object({
|
||||
boot_disk_type = string
|
||||
boot_disk_size_gb = number
|
||||
num_local_ssds = number
|
||||
}), null)
|
||||
image_uri = string
|
||||
accelerators = optional(object({
|
||||
accelerator_type = string
|
||||
accelerator_count = number
|
||||
}), null)
|
||||
}), null)
|
||||
preemptible_worker_config = optional(object({
|
||||
num_instances = number
|
||||
preemptibility = string
|
||||
disk_config = optional(object({
|
||||
boot_disk_type = string
|
||||
boot_disk_size_gb = number
|
||||
num_local_ssds = number
|
||||
}), null)
|
||||
}), null)
|
||||
software_config = optional(object({
|
||||
image_version = string
|
||||
override_properties = list(map(string))
|
||||
optional_components = list(string)
|
||||
}), null)
|
||||
security_config = optional(object({
|
||||
kerberos_config = object({
|
||||
cross_realm_trust_admin_server = optional(string, null)
|
||||
cross_realm_trust_kdc = optional(string, null)
|
||||
cross_realm_trust_realm = optional(string, null)
|
||||
cross_realm_trust_shared_password_uri = optional(string, null)
|
||||
enable_kerberos = optional(string, null)
|
||||
kdc_db_key_uri = optional(string, null)
|
||||
key_password_uri = optional(string, null)
|
||||
keystore_uri = optional(string, null)
|
||||
keystore_password_uri = optional(string, null)
|
||||
kms_key_uri = string
|
||||
realm = optional(string, null)
|
||||
root_principal_password_uri = string
|
||||
tgt_lifetime_hours = optional(string, null)
|
||||
truststore_password_uri = optional(string, null)
|
||||
truststore_uri = optional(string, null)
|
||||
})
|
||||
}), null)
|
||||
autoscaling_config = optional(object({
|
||||
policy_uri = string
|
||||
}), null)
|
||||
initialization_action = optional(object({
|
||||
script = string
|
||||
timeout_sec = optional(string, null)
|
||||
}), null)
|
||||
encryption_config = optional(object({
|
||||
kms_key_name = string
|
||||
}), null)
|
||||
lifecycle_config = optional(object({
|
||||
idle_delete_ttl = optional(string, null)
|
||||
auto_delete_time = optional(string, null)
|
||||
}), null)
|
||||
endpoint_config = optional(object({
|
||||
enable_http_port_access = string
|
||||
}), null)
|
||||
dataproc_metric_config = optional(object({
|
||||
metrics = list(object({
|
||||
metric_source = string
|
||||
metric_overrides = optional(string, null)
|
||||
}))
|
||||
}), null)
|
||||
metastore_config = optional(object({
|
||||
dataproc_metastore_service = string
|
||||
}), null)
|
||||
}), null)
|
||||
|
||||
virtual_cluster_config = optional(object({
|
||||
staging_bucket = optional(string, null)
|
||||
auxiliary_services_config = optional(object({
|
||||
metastore_config = optional(object({
|
||||
dataproc_metastore_service = string
|
||||
}), null)
|
||||
spark_history_server_config = optional(object({
|
||||
dataproc_cluster = string
|
||||
}), null)
|
||||
}), null)
|
||||
kubernetes_cluster_config = object({
|
||||
kubernetes_namespace = optional(string, null)
|
||||
kubernetes_software_config = object({
|
||||
component_version = list(map(string))
|
||||
properties = optional(list(map(string)), null)
|
||||
})
|
||||
|
||||
gke_cluster_config = object({
|
||||
gke_cluster_target = optional(string, null)
|
||||
node_pool_target = optional(object({
|
||||
node_pool = string
|
||||
roles = list(string)
|
||||
node_pool_config = optional(object({
|
||||
autoscaling = optional(object({
|
||||
min_node_count = optional(number, null)
|
||||
max_node_count = optional(number, null)
|
||||
}), null)
|
||||
|
||||
config = object({
|
||||
machine_type = optional(string, null)
|
||||
preemptible = optional(bool, null)
|
||||
local_ssd_count = optional(number, null)
|
||||
min_cpu_platform = optional(string, null)
|
||||
spot = optional(bool, null)
|
||||
})
|
||||
|
||||
locations = optional(list(string), null)
|
||||
}), null)
|
||||
}), null)
|
||||
})
|
||||
})
|
||||
}), null)
|
||||
})
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "group_iam" {
|
||||
description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable."
|
||||
type = map(list(string))
|
||||
default = {}
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "iam" {
|
||||
description = "IAM bindings in {ROLE => [MEMBERS]} format."
|
||||
type = map(list(string))
|
||||
default = {}
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "iam_additive" {
|
||||
description = "IAM additive bindings in {ROLE => [MEMBERS]} format."
|
||||
type = map(list(string))
|
||||
default = {}
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "labels" {
|
||||
description = "The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs."
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "name" {
|
||||
description = "Cluster name."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "prefix" {
|
||||
description = "Optional prefix used to generate project id and name."
|
||||
type = string
|
||||
default = null
|
||||
validation {
|
||||
condition = var.prefix != ""
|
||||
error_message = "Prefix cannot be empty, please use null instead."
|
||||
}
|
||||
}
|
||||
|
||||
variable "project_id" {
|
||||
description = "Project ID."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "region" {
|
||||
description = "Dataproc region."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "service_account" {
|
||||
description = "Service account to set on the Dataproc cluster."
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
terraform {
|
||||
required_version = ">= 1.3.1"
|
||||
required_providers {
|
||||
google = {
|
||||
source = "hashicorp/google"
|
||||
version = ">= 4.50.0" # tftest
|
||||
}
|
||||
google-beta = {
|
||||
source = "hashicorp/google-beta"
|
||||
version = ">= 4.50.0" # tftest
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue