Merge pull request #1075 from iht/bigtable_replication
Add cluster replicas to Bigtable module.
This commit is contained in:
Israel Herraiz
GitHub
commit
0380c1004b
|
|
@ -5,7 +5,6 @@ This module allows managing a single BigTable instance, including access configu
|
|||
## TODO
|
||||
|
||||
- [ ] support bigtable_app_profile
|
||||
- [ ] support cluster replicas
|
||||
- [ ] support IAM for tables
|
||||
|
||||
## Examples
|
||||
|
|
@ -18,8 +17,11 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
}
|
||||
tables = {
|
||||
test1 = {},
|
||||
test2 = {
|
||||
|
|
@ -41,8 +43,11 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
}
|
||||
tables = {
|
||||
test1 = {},
|
||||
test2 = {
|
||||
|
|
@ -63,6 +68,29 @@ module "bigtable-instance" {
|
|||
# tftest modules=1 resources=4
|
||||
```
|
||||
|
||||
### Instance with replication enabled
|
||||
|
||||
```hcl
|
||||
|
||||
module "bigtable-instance" {
|
||||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
clusters = {
|
||||
first-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
second-cluster = {
|
||||
zone = "europe-southwest1-a"
|
||||
}
|
||||
third-cluster = {
|
||||
zone = "us-central1-b"
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=1
|
||||
```
|
||||
|
||||
### Instance with garbage collection policy
|
||||
|
||||
```hcl
|
||||
|
|
@ -71,8 +99,11 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
}
|
||||
tables = {
|
||||
test1 = {
|
||||
column_families = {
|
||||
|
|
@ -102,8 +133,11 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
}
|
||||
}
|
||||
default_gc_policy = {
|
||||
deletion_policy = "ABANDON"
|
||||
max_age = "18h"
|
||||
|
|
@ -131,9 +165,12 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
num_nodes = 5
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-west1-b"
|
||||
num_nodes = 5
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=1
|
||||
```
|
||||
|
|
@ -148,13 +185,18 @@ module "bigtable-instance" {
|
|||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-southwest1-b"
|
||||
autoscaling_config = {
|
||||
min_nodes = 3
|
||||
max_nodes = 7
|
||||
cpu_target = 70
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-southwest1-b"
|
||||
autoscaling = {
|
||||
min_nodes = 3
|
||||
max_nodes = 7
|
||||
cpu_target = 70
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
# tftest modules=1 resources=1
|
||||
```
|
||||
|
|
@ -164,17 +206,20 @@ module "bigtable-instance" {
|
|||
```hcl
|
||||
|
||||
module "bigtable-instance" {
|
||||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-southwest1-a"
|
||||
storage_type = "SSD"
|
||||
autoscaling_config = {
|
||||
min_nodes = 3
|
||||
max_nodes = 7
|
||||
cpu_target = 70
|
||||
storage_target = 4096
|
||||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
clusters = {
|
||||
my-cluster = {
|
||||
zone = "europe-southwest1-a"
|
||||
storage_type = "SSD"
|
||||
autoscaling = {
|
||||
min_nodes = 3
|
||||
max_nodes = 7
|
||||
cpu_target = 70
|
||||
storage_target = 4096
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=1
|
||||
|
|
@ -185,19 +230,16 @@ module "bigtable-instance" {
|
|||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [name](variables.tf#L68) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L79) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
|
||||
| [zone](variables.tf#L109) | The zone to create the Cloud Bigtable cluster in. | <code>string</code> | ✓ | |
|
||||
| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | <code title="object({ min_nodes = number max_nodes = number cpu_target = number storage_target = optional(number, null) })">object({…})</code> | | <code>null</code> |
|
||||
| [cluster_id](variables.tf#L28) | The ID of the Cloud Bigtable cluster. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [default_gc_policy](variables.tf#L34) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||
| [deletion_protection](variables.tf#L46) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
|
||||
| [display_name](variables.tf#L51) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
|
||||
| [iam](variables.tf#L56) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [instance_type](variables.tf#L62) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
|
||||
| [num_nodes](variables.tf#L73) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | <code>number</code> | | <code>1</code> |
|
||||
| [storage_type](variables.tf#L84) | The storage type to use. | <code>string</code> | | <code>"SSD"</code> |
|
||||
| [tables](variables.tf#L90) | Tables to be created in the BigTable instance. | <code title="map(object({ split_keys = optional(list(string), []) column_families = optional(map(object( { gc_policy = optional(object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) }), null) })), {}) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [clusters](variables.tf#L17) | Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored. | <code title="map(object({ zone = optional(string) storage_type = optional(string) num_nodes = optional(number) autoscaling = optional(object({ min_nodes = number max_nodes = number cpu_target = number storage_target = optional(number) })) }))">map(object({…}))</code> | ✓ | |
|
||||
| [name](variables.tf#L78) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L83) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
|
||||
| [default_autoscaling](variables.tf#L33) | Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details. | <code title="object({ min_nodes = number max_nodes = number cpu_target = number storage_target = optional(number) })">object({…})</code> | | <code>null</code> |
|
||||
| [default_gc_policy](variables.tf#L44) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||
| [deletion_protection](variables.tf#L56) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
|
||||
| [display_name](variables.tf#L61) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
|
||||
| [iam](variables.tf#L66) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [instance_type](variables.tf#L72) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
|
||||
| [tables](variables.tf#L88) | Tables to be created in the BigTable instance. | <code title="map(object({ split_keys = optional(list(string), []) column_families = optional(map(object( { gc_policy = optional(object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) }), null) })), {}) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
*/
|
||||
|
||||
locals {
|
||||
num_nodes = var.autoscaling_config == null ? var.num_nodes : null
|
||||
gc_pairs = flatten([
|
||||
for table, table_obj in var.tables : [
|
||||
for cf, cf_obj in table_obj.column_families : {
|
||||
|
|
@ -25,30 +24,44 @@ locals {
|
|||
}
|
||||
]
|
||||
])
|
||||
|
||||
clusters_autoscaling = {
|
||||
for cluster_id, cluster in var.clusters : cluster_id => {
|
||||
zone = cluster.zone
|
||||
storage_type = cluster.storage_type
|
||||
num_nodes = cluster.autoscaling == null && var.default_autoscaling == null ? cluster.num_nodes : null
|
||||
autoscaling = cluster.autoscaling == null ? var.default_autoscaling : cluster.autoscaling
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_bigtable_instance" "default" {
|
||||
project = var.project_id
|
||||
name = var.name
|
||||
cluster {
|
||||
cluster_id = var.cluster_id
|
||||
zone = var.zone
|
||||
storage_type = var.storage_type
|
||||
num_nodes = local.num_nodes
|
||||
dynamic "autoscaling_config" {
|
||||
for_each = var.autoscaling_config == null ? [] : [""]
|
||||
content {
|
||||
min_nodes = var.autoscaling_config.min_nodes
|
||||
max_nodes = var.autoscaling_config.max_nodes
|
||||
cpu_target = var.autoscaling_config.cpu_target
|
||||
storage_target = var.autoscaling_config.storage_target
|
||||
|
||||
instance_type = var.instance_type
|
||||
display_name = var.display_name == null ? var.display_name : var.name
|
||||
deletion_protection = var.deletion_protection
|
||||
|
||||
dynamic "cluster" {
|
||||
for_each = local.clusters_autoscaling
|
||||
content {
|
||||
cluster_id = cluster.key
|
||||
zone = cluster.value.zone
|
||||
storage_type = cluster.value.storage_type
|
||||
num_nodes = cluster.value.num_nodes
|
||||
|
||||
dynamic "autoscaling_config" {
|
||||
for_each = cluster.value.autoscaling == null ? [] : [""]
|
||||
content {
|
||||
min_nodes = cluster.value.autoscaling.min_nodes
|
||||
max_nodes = cluster.value.autoscaling.max_nodes
|
||||
cpu_target = cluster.value.autoscaling.cpu_target
|
||||
storage_target = cluster.value.autoscaling.storage_target
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
instance_type = var.instance_type
|
||||
|
||||
display_name = var.display_name == null ? var.display_name : var.name
|
||||
deletion_protection = var.deletion_protection
|
||||
}
|
||||
|
||||
resource "google_bigtable_instance_iam_binding" "default" {
|
||||
|
|
|
|||
|
|
@ -14,23 +14,33 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
variable "autoscaling_config" {
|
||||
description = "Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored."
|
||||
variable "clusters" {
|
||||
description = "Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored."
|
||||
nullable = false
|
||||
type = map(object({
|
||||
zone = optional(string)
|
||||
storage_type = optional(string)
|
||||
num_nodes = optional(number)
|
||||
autoscaling = optional(object({
|
||||
min_nodes = number
|
||||
max_nodes = number
|
||||
cpu_target = number
|
||||
storage_target = optional(number)
|
||||
}))
|
||||
}))
|
||||
}
|
||||
|
||||
variable "default_autoscaling" {
|
||||
description = "Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details."
|
||||
type = object({
|
||||
min_nodes = number
|
||||
max_nodes = number
|
||||
cpu_target = number
|
||||
storage_target = optional(number, null)
|
||||
storage_target = optional(number)
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "cluster_id" {
|
||||
description = "The ID of the Cloud Bigtable cluster."
|
||||
type = string
|
||||
default = "europe-west1"
|
||||
}
|
||||
|
||||
variable "default_gc_policy" {
|
||||
description = "Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families."
|
||||
type = object({
|
||||
|
|
@ -70,23 +80,11 @@ variable "name" {
|
|||
type = string
|
||||
}
|
||||
|
||||
variable "num_nodes" {
|
||||
description = "The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling."
|
||||
type = number
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "project_id" {
|
||||
description = "Id of the project where datasets will be created."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "storage_type" {
|
||||
description = "The storage type to use."
|
||||
type = string
|
||||
default = "SSD"
|
||||
}
|
||||
|
||||
variable "tables" {
|
||||
description = "Tables to be created in the BigTable instance."
|
||||
nullable = false
|
||||
|
|
@ -105,8 +103,3 @@ variable "tables" {
|
|||
}))
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "zone" {
|
||||
description = "The zone to create the Cloud Bigtable cluster in."
|
||||
type = string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,5 +28,9 @@ module "test" {
|
|||
}
|
||||
|
||||
}
|
||||
zone = var.zone
|
||||
clusters = {
|
||||
test = {
|
||||
zone = var.zone
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue