Add cluster replicas to Bigtable module.
This adds the possiblity to define several clusters in a Bigtable instance, which enables replication in Bigtable. Some of the configurations options are moved inside a `map` that defines the properties of each cluster. These changes alter the interface of the module, so any previous code using this module will have to adapt to the new options interface.
This commit is contained in:
Israel Herraiz
parent
39242c3947
commit
20579394b8
|
|
@ -5,7 +5,6 @@ This module allows managing a single BigTable instance, including access configu
|
|||
## TODO
|
||||
|
||||
- [ ] support bigtable_app_profile
|
||||
- [ ] support cluster replicas
|
||||
- [ ] support IAM for tables
|
||||
|
||||
## Examples
|
||||
|
|
@ -18,8 +17,11 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
}
|
||||
tables = {
|
||||
test1 = {},
|
||||
test2 = {
|
||||
|
|
@ -41,8 +43,11 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
}
|
||||
tables = {
|
||||
test1 = {},
|
||||
test2 = {
|
||||
|
|
@ -63,6 +68,29 @@ module "bigtable-instance" {
|
|||
# tftest modules=1 resources=4
|
||||
```
|
||||
|
||||
### Instance with replication enabled
|
||||
|
||||
```hcl
|
||||
|
||||
module "bigtable-instance" {
|
||||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
clusters = {
|
||||
first-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
second-cluster = {
|
||||
zone = "europe-southwest1-a"
|
||||
}
|
||||
third-cluster = {
|
||||
zone = "us-central1-b"
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=1
|
||||
```
|
||||
|
||||
### Instance with garbage collection policy
|
||||
|
||||
```hcl
|
||||
|
|
@ -71,8 +99,11 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
}
|
||||
tables = {
|
||||
test1 = {
|
||||
column_families = {
|
||||
|
|
@ -102,8 +133,11 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
}
|
||||
default_gc_policy = {
|
||||
deletion_policy = "ABANDON"
|
||||
max_age = "18h"
|
||||
|
|
@ -131,9 +165,12 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
num_nodes = 5
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
num_nodes = 5
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=1
|
||||
```
|
||||
|
|
@ -148,13 +185,18 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-southwest1-b"
|
||||
autoscaling_config = {
|
||||
min_nodes = 3
|
||||
max_nodes = 7
|
||||
cpu_target = 70
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-southwest1-b"
|
||||
autoscaling = {
|
||||
min_nodes = 3
|
||||
max_nodes = 7
|
||||
cpu_target = 70
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
# tftest modules=1 resources=1
|
||||
```
|
||||
|
|
@ -164,17 +206,20 @@ module "bigtable-instance" {
|
|||
```hcl
|
||||
|
||||
module "bigtable-instance" {
|
||||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-southwest1-a"
|
||||
storage_type = "SSD"
|
||||
autoscaling_config = {
|
||||
min_nodes = 3
|
||||
max_nodes = 7
|
||||
cpu_target = 70
|
||||
storage_target = 4096
|
||||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-southwest1-a"
|
||||
storage_type = "SSD"
|
||||
autoscaling = {
|
||||
min_nodes = 3
|
||||
max_nodes = 7
|
||||
cpu_target = 70
|
||||
storage_target = 4096
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=1
|
||||
|
|
@ -185,19 +230,16 @@ module "bigtable-instance" {
|
|||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [name](variables.tf#L68) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L79) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
|
||||
| [zone](variables.tf#L109) | The zone to create the Cloud Bigtable cluster in. | <code>string</code> | ✓ | |
|
||||
| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | <code title="object({ min_nodes = number max_nodes = number cpu_target = number storage_target = optional(number, null) })">object({…})</code> | | <code>null</code> |
|
||||
| [cluster_id](variables.tf#L28) | The ID of the Cloud Bigtable cluster. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [default_gc_policy](variables.tf#L34) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||
| [deletion_protection](variables.tf#L46) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
|
||||
| [display_name](variables.tf#L51) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
|
||||
| [iam](variables.tf#L56) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [instance_type](variables.tf#L62) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
|
||||
| [num_nodes](variables.tf#L73) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | <code>number</code> | | <code>1</code> |
|
||||
| [storage_type](variables.tf#L84) | The storage type to use. | <code>string</code> | | <code>"SSD"</code> |
|
||||
| [tables](variables.tf#L90) | Tables to be created in the BigTable instance. | <code title="map(object({ split_keys = optional(list(string), []) column_families = optional(map(object( { gc_policy = optional(object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) }), null) })), {}) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [clusters](variables.tf#L17) | Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored. | <code title="map(object({ zone = optional(string) storage_type = optional(string) num_nodes = optional(number) autoscaling = optional(object({ min_nodes = number max_nodes = number cpu_target = number storage_target = optional(number) })) }))">map(object({…}))</code> | ✓ | |
|
||||
| [name](variables.tf#L78) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L83) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
|
||||
| [default_autoscaling](variables.tf#L33) | Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details. | <code title="object({ min_nodes = number max_nodes = number cpu_target = number storage_target = optional(number) })">object({…})</code> | | <code>null</code> |
|
||||
| [default_gc_policy](variables.tf#L44) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||
| [deletion_protection](variables.tf#L56) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
|
||||
| [display_name](variables.tf#L61) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
|
||||
| [iam](variables.tf#L66) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [instance_type](variables.tf#L72) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
|
||||
| [tables](variables.tf#L88) | Tables to be created in the BigTable instance. | <code title="map(object({ split_keys = optional(list(string), []) column_families = optional(map(object( { gc_policy = optional(object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) }), null) })), {}) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
*/
|
||||
|
||||
locals {
|
||||
num_nodes = var.autoscaling_config == null ? var.num_nodes : null
|
||||
gc_pairs = flatten([
|
||||
for table, table_obj in var.tables : [
|
||||
for cf, cf_obj in table_obj.column_families : {
|
||||
|
|
@ -25,30 +24,45 @@ locals {
|
|||
}
|
||||
]
|
||||
])
|
||||
|
||||
clusters_autoscaling = flatten([
|
||||
for cluster, obj in var.clusters : [{
|
||||
cluster_id = cluster
|
||||
zone = obj.zone
|
||||
storage_type = obj.storage_type
|
||||
num_nodes = obj.autoscaling == null && var.default_autoscaling == null ? obj.num_nodes : null
|
||||
autoscaling = obj.autoscaling == null ? var.default_autoscaling : obj.autoscaling
|
||||
}]
|
||||
])
|
||||
}
|
||||
|
||||
resource "google_bigtable_instance" "default" {
|
||||
project = var.project_id
|
||||
name = var.name
|
||||
cluster {
|
||||
cluster_id = var.cluster_id
|
||||
zone = var.zone
|
||||
storage_type = var.storage_type
|
||||
num_nodes = local.num_nodes
|
||||
dynamic "autoscaling_config" {
|
||||
for_each = var.autoscaling_config == null ? [] : [""]
|
||||
content {
|
||||
min_nodes = var.autoscaling_config.min_nodes
|
||||
max_nodes = var.autoscaling_config.max_nodes
|
||||
cpu_target = var.autoscaling_config.cpu_target
|
||||
storage_target = var.autoscaling_config.storage_target
|
||||
|
||||
instance_type = var.instance_type
|
||||
display_name = var.display_name == null ? var.display_name : var.name
|
||||
deletion_protection = var.deletion_protection
|
||||
|
||||
dynamic "cluster" {
|
||||
for_each = { for k, v in local.clusters_autoscaling : k => v }
|
||||
content {
|
||||
cluster_id = cluster.value.cluster_id
|
||||
zone = cluster.value.zone
|
||||
storage_type = cluster.value.storage_type
|
||||
num_nodes = cluster.value.num_nodes
|
||||
|
||||
dynamic "autoscaling_config" {
|
||||
for_each = cluster.value.autoscaling == null ? [] : [""]
|
||||
content {
|
||||
min_nodes = cluster.value.autoscaling.min_nodes
|
||||
max_nodes = cluster.value.autoscaling.max_nodes
|
||||
cpu_target = cluster.value.autoscaling.cpu_target
|
||||
storage_target = cluster.value.autoscaling.storage_target
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
instance_type = var.instance_type
|
||||
|
||||
display_name = var.display_name == null ? var.display_name : var.name
|
||||
deletion_protection = var.deletion_protection
|
||||
}
|
||||
|
||||
resource "google_bigtable_instance_iam_binding" "default" {
|
||||
|
|
|
|||
|
|
@ -14,23 +14,33 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
variable "autoscaling_config" {
|
||||
description = "Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored."
|
||||
variable "clusters" {
|
||||
description = "Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored."
|
||||
nullable = false
|
||||
type = map(object({
|
||||
zone = optional(string)
|
||||
storage_type = optional(string)
|
||||
num_nodes = optional(number)
|
||||
autoscaling = optional(object({
|
||||
min_nodes = number
|
||||
max_nodes = number
|
||||
cpu_target = number
|
||||
storage_target = optional(number)
|
||||
}))
|
||||
}))
|
||||
}
|
||||
|
||||
variable "default_autoscaling" {
|
||||
description = "Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details."
|
||||
type = object({
|
||||
min_nodes = number
|
||||
max_nodes = number
|
||||
cpu_target = number
|
||||
storage_target = optional(number, null)
|
||||
storage_target = optional(number)
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "cluster_id" {
|
||||
description = "The ID of the Cloud Bigtable cluster."
|
||||
type = string
|
||||
default = "europe-west1"
|
||||
}
|
||||
|
||||
variable "default_gc_policy" {
|
||||
description = "Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families."
|
||||
type = object({
|
||||
|
|
@ -70,23 +80,11 @@ variable "name" {
|
|||
type = string
|
||||
}
|
||||
|
||||
variable "num_nodes" {
|
||||
description = "The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling."
|
||||
type = number
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "project_id" {
|
||||
description = "Id of the project where datasets will be created."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "storage_type" {
|
||||
description = "The storage type to use."
|
||||
type = string
|
||||
default = "SSD"
|
||||
}
|
||||
|
||||
variable "tables" {
|
||||
description = "Tables to be created in the BigTable instance."
|
||||
nullable = false
|
||||
|
|
@ -105,8 +103,3 @@ variable "tables" {
|
|||
}))
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "zone" {
|
||||
description = "The zone to create the Cloud Bigtable cluster in."
|
||||
type = string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,5 +28,9 @@ module "test" {
|
|||
}
|
||||
|
||||
}
|
||||
zone = var.zone
|
||||
clusters = {
|
||||
test = {
|
||||
zone = var.zone
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue