From 20579394b877bfcdde1d2bfd1d6904ee7e470165 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Wed, 28 Dec 2022 19:53:17 +0100 Subject: [PATCH] Add cluster replicas to Bigtable module. This adds the possiblity to define several clusters in a Bigtable instance, which enables replication in Bigtable. Some of the configurations options are moved inside a `map` that defines the properties of each cluster. These changes alter the interface of the module, so any previous code using this module will have to adapt to the new options interface. --- modules/bigtable-instance/README.md | 126 ++++++++++++------ modules/bigtable-instance/main.tf | 48 ++++--- modules/bigtable-instance/variables.tf | 45 +++---- .../modules/bigtable_instance/fixture/main.tf | 6 +- 4 files changed, 139 insertions(+), 86 deletions(-) diff --git a/modules/bigtable-instance/README.md b/modules/bigtable-instance/README.md index b72f8fec..39e0bc53 100644 --- a/modules/bigtable-instance/README.md +++ b/modules/bigtable-instance/README.md @@ -5,7 +5,6 @@ This module allows managing a single BigTable instance, including access configu ## TODO - [ ] support bigtable_app_profile -- [ ] support cluster replicas - [ ] support IAM for tables ## Examples @@ -18,8 +17,11 @@ module "bigtable-instance" { source = "./fabric/modules/bigtable-instance" project_id = "my-project" name = "instance" - cluster_id = "instance" - zone = "europe-west1-b" + clusters = { + my-cluster = { + zone = "europe-west1-b" + } + } tables = { test1 = {}, test2 = { @@ -41,8 +43,11 @@ module "bigtable-instance" { source = "./fabric/modules/bigtable-instance" project_id = "my-project" name = "instance" - cluster_id = "instance" - zone = "europe-west1-b" + clusters = { + my-cluster = { + zone = "europe-west1-b" + } + } tables = { test1 = {}, test2 = { @@ -63,6 +68,29 @@ module "bigtable-instance" { # tftest modules=1 resources=4 ``` +### Instance with replication enabled + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + first-cluster = { + zone = "europe-west1-b" + } + second-cluster = { + zone = "europe-southwest1-a" + } + third-cluster = { + zone = "us-central1-b" + } + } +} +# tftest modules=1 resources=1 +``` + ### Instance with garbage collection policy ```hcl @@ -71,8 +99,11 @@ module "bigtable-instance" { source = "./fabric/modules/bigtable-instance" project_id = "my-project" name = "instance" - cluster_id = "instance" - zone = "europe-west1-b" + clusters = { + my-cluster = { + zone = "europe-west1-b" + } + } tables = { test1 = { column_families = { @@ -102,8 +133,11 @@ module "bigtable-instance" { source = "./fabric/modules/bigtable-instance" project_id = "my-project" name = "instance" - cluster_id = "instance" - zone = "europe-west1-b" + clusters = { + my-cluster = { + zone = "europe-west1-b" + } + } default_gc_policy = { deletion_policy = "ABANDON" max_age = "18h" @@ -131,9 +165,12 @@ module "bigtable-instance" { source = "./fabric/modules/bigtable-instance" project_id = "my-project" name = "instance" - cluster_id = "instance" - zone = "europe-west1-b" - num_nodes = 5 + clusters = { + my-cluster = { + zone = "europe-west1-b" + num_nodes = 5 + } + } } # tftest modules=1 resources=1 ``` @@ -148,13 +185,18 @@ module "bigtable-instance" { source = "./fabric/modules/bigtable-instance" project_id = "my-project" name = "instance" - cluster_id = "instance" - zone = "europe-southwest1-b" - autoscaling_config = { - min_nodes = 3 - max_nodes = 7 - cpu_target = 70 + clusters = { + my-cluster = { + zone = "europe-southwest1-b" + autoscaling = { + min_nodes = 3 + max_nodes = 7 + cpu_target = 70 + } + } } + + } # tftest modules=1 resources=1 ``` @@ -164,17 +206,20 @@ module "bigtable-instance" { ```hcl module "bigtable-instance" { - source = "./fabric/modules/bigtable-instance" - project_id = "my-project" - name = "instance" - cluster_id = "instance" - zone = "europe-southwest1-a" - storage_type = "SSD" - autoscaling_config = { - min_nodes = 3 - max_nodes = 7 - cpu_target = 70 - storage_target = 4096 + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + my-cluster = { + zone = "europe-southwest1-a" + storage_type = "SSD" + autoscaling = { + min_nodes = 3 + max_nodes = 7 + cpu_target = 70 + storage_target = 4096 + } + } } } # tftest modules=1 resources=1 @@ -185,19 +230,16 @@ module "bigtable-instance" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L68) | The name of the Cloud Bigtable instance. | string | ✓ | | -| [project_id](variables.tf#L79) | Id of the project where datasets will be created. | string | ✓ | | -| [zone](variables.tf#L109) | The zone to create the Cloud Bigtable cluster in. | string | ✓ | | -| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | object({…}) | | null | -| [cluster_id](variables.tf#L28) | The ID of the Cloud Bigtable cluster. | string | | "europe-west1" | -| [default_gc_policy](variables.tf#L34) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | object({…}) | | null | -| [deletion_protection](variables.tf#L46) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | | | true | -| [display_name](variables.tf#L51) | The human-readable display name of the Bigtable instance. | | | null | -| [iam](variables.tf#L56) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [instance_type](variables.tf#L62) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | string | | null | -| [num_nodes](variables.tf#L73) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | number | | 1 | -| [storage_type](variables.tf#L84) | The storage type to use. | string | | "SSD" | -| [tables](variables.tf#L90) | Tables to be created in the BigTable instance. | map(object({…})) | | {} | +| [clusters](variables.tf#L17) | Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored. | map(object({…})) | ✓ | | +| [name](variables.tf#L78) | The name of the Cloud Bigtable instance. | string | ✓ | | +| [project_id](variables.tf#L83) | Id of the project where datasets will be created. | string | ✓ | | +| [default_autoscaling](variables.tf#L33) | Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details. | object({…}) | | null | +| [default_gc_policy](variables.tf#L44) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | object({…}) | | null | +| [deletion_protection](variables.tf#L56) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | | | true | +| [display_name](variables.tf#L61) | The human-readable display name of the Bigtable instance. | | | null | +| [iam](variables.tf#L66) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [instance_type](variables.tf#L72) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | string | | null | +| [tables](variables.tf#L88) | Tables to be created in the BigTable instance. | map(object({…})) | | {} | ## Outputs diff --git a/modules/bigtable-instance/main.tf b/modules/bigtable-instance/main.tf index e1ab9cf3..8eb142a7 100644 --- a/modules/bigtable-instance/main.tf +++ b/modules/bigtable-instance/main.tf @@ -15,7 +15,6 @@ */ locals { - num_nodes = var.autoscaling_config == null ? var.num_nodes : null gc_pairs = flatten([ for table, table_obj in var.tables : [ for cf, cf_obj in table_obj.column_families : { @@ -25,30 +24,45 @@ locals { } ] ]) + + clusters_autoscaling = flatten([ + for cluster, obj in var.clusters : [{ + cluster_id = cluster + zone = obj.zone + storage_type = obj.storage_type + num_nodes = obj.autoscaling == null && var.default_autoscaling == null ? obj.num_nodes : null + autoscaling = obj.autoscaling == null ? var.default_autoscaling : obj.autoscaling + }] + ]) } resource "google_bigtable_instance" "default" { project = var.project_id name = var.name - cluster { - cluster_id = var.cluster_id - zone = var.zone - storage_type = var.storage_type - num_nodes = local.num_nodes - dynamic "autoscaling_config" { - for_each = var.autoscaling_config == null ? [] : [""] - content { - min_nodes = var.autoscaling_config.min_nodes - max_nodes = var.autoscaling_config.max_nodes - cpu_target = var.autoscaling_config.cpu_target - storage_target = var.autoscaling_config.storage_target + + instance_type = var.instance_type + display_name = var.display_name == null ? var.display_name : var.name + deletion_protection = var.deletion_protection + + dynamic "cluster" { + for_each = { for k, v in local.clusters_autoscaling : k => v } + content { + cluster_id = cluster.value.cluster_id + zone = cluster.value.zone + storage_type = cluster.value.storage_type + num_nodes = cluster.value.num_nodes + + dynamic "autoscaling_config" { + for_each = cluster.value.autoscaling == null ? [] : [""] + content { + min_nodes = cluster.value.autoscaling.min_nodes + max_nodes = cluster.value.autoscaling.max_nodes + cpu_target = cluster.value.autoscaling.cpu_target + storage_target = cluster.value.autoscaling.storage_target + } } } } - instance_type = var.instance_type - - display_name = var.display_name == null ? var.display_name : var.name - deletion_protection = var.deletion_protection } resource "google_bigtable_instance_iam_binding" "default" { diff --git a/modules/bigtable-instance/variables.tf b/modules/bigtable-instance/variables.tf index 1ccd969b..f7b75c13 100644 --- a/modules/bigtable-instance/variables.tf +++ b/modules/bigtable-instance/variables.tf @@ -14,23 +14,33 @@ * limitations under the License. */ -variable "autoscaling_config" { - description = "Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored." +variable "clusters" { + description = "Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored." + nullable = false + type = map(object({ + zone = optional(string) + storage_type = optional(string) + num_nodes = optional(number) + autoscaling = optional(object({ + min_nodes = number + max_nodes = number + cpu_target = number + storage_target = optional(number) + })) + })) +} + +variable "default_autoscaling" { + description = "Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details." type = object({ min_nodes = number max_nodes = number cpu_target = number - storage_target = optional(number, null) + storage_target = optional(number) }) default = null } -variable "cluster_id" { - description = "The ID of the Cloud Bigtable cluster." - type = string - default = "europe-west1" -} - variable "default_gc_policy" { description = "Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families." type = object({ @@ -70,23 +80,11 @@ variable "name" { type = string } -variable "num_nodes" { - description = "The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling." - type = number - default = 1 -} - variable "project_id" { description = "Id of the project where datasets will be created." type = string } -variable "storage_type" { - description = "The storage type to use." - type = string - default = "SSD" -} - variable "tables" { description = "Tables to be created in the BigTable instance." nullable = false @@ -105,8 +103,3 @@ variable "tables" { })) default = {} } - -variable "zone" { - description = "The zone to create the Cloud Bigtable cluster in." - type = string -} diff --git a/tests/modules/bigtable_instance/fixture/main.tf b/tests/modules/bigtable_instance/fixture/main.tf index c7a47770..4fa83ce2 100644 --- a/tests/modules/bigtable_instance/fixture/main.tf +++ b/tests/modules/bigtable_instance/fixture/main.tf @@ -28,5 +28,9 @@ module "test" { } } - zone = var.zone + clusters = { + test = { + zone = var.zone + } + } }