Add gc_policy to Bigtable module.
Column families have now a new property, to specify the garbage collection policy. A new option also allows to set a default policy if none is specified. This changes the previous syntax for column families, that was helpful since the policy is column-family specific, the new syntax makes it easier to specify a policy per column family.
This commit is contained in:
Israel Herraiz
parent
008828f2d8
commit
32bee7104e
|
|
@ -17,11 +17,11 @@ terraform {
|
|||
required_providers {
|
||||
google = {
|
||||
source = "hashicorp/google"
|
||||
version = ">= 4.40.0" # tftest
|
||||
version = ">= 4.47.0" # tftest
|
||||
}
|
||||
google-beta = {
|
||||
source = "hashicorp/google-beta"
|
||||
version = ">= 4.40.0" # tftest
|
||||
version = ">= 4.47.0" # tftest
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ This module allows managing a single BigTable instance, including access configu
|
|||
|
||||
## TODO
|
||||
|
||||
- [ ] support bigtable_gc_policy
|
||||
- [ ] support bigtable_app_profile
|
||||
- [ ] support cluster replicas
|
||||
- [ ] support IAM for tables
|
||||
|
|
@ -47,11 +46,75 @@ module "bigtable-instance" {
|
|||
tables = {
|
||||
test1 = {},
|
||||
test2 = {
|
||||
split_keys = ["a", "b", "c"]
|
||||
column_families = ["cf1", "cf2", "cf3"]
|
||||
split_keys = ["a", "b", "c"]
|
||||
column_families = {
|
||||
cf1 = {}
|
||||
cf2 = {}
|
||||
cf3 = {}
|
||||
}
|
||||
}
|
||||
test3 = {
|
||||
column_families = ["cf1"]
|
||||
column_families = {
|
||||
cf1 = {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=4
|
||||
```
|
||||
|
||||
### Instance with garbage collection policy
|
||||
|
||||
```hcl
|
||||
|
||||
module "bigtable-instance" {
|
||||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
tables = {
|
||||
test1 = {
|
||||
column_families = {
|
||||
cf1 = {
|
||||
gc_policy = {
|
||||
deletion_policy = "ABANDON"
|
||||
max_age = "18h"
|
||||
}
|
||||
}
|
||||
cf2 = {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
# tftest modules=1 resources=3
|
||||
```
|
||||
|
||||
### Instance with default garbage collection policy
|
||||
|
||||
The default garbage collection policy is applied to any column family that does
|
||||
not specify a `gc_policy`. If a column family specifies a `gc_policy`, the
|
||||
default garbage collection policy is ignored for that column family.
|
||||
|
||||
```hcl
|
||||
|
||||
module "bigtable-instance" {
|
||||
source = "./fabric/modules/bigtable-instance"
|
||||
project_id = "my-project"
|
||||
name = "instance"
|
||||
cluster_id = "instance"
|
||||
zone = "europe-west1-b"
|
||||
default_gc_policy = {
|
||||
deletion_policy = "ABANDON"
|
||||
max_age = "18h"
|
||||
max_version = 7
|
||||
}
|
||||
tables = {
|
||||
test1 = {
|
||||
column_families = {
|
||||
cf1 = {}
|
||||
cf2 = {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -122,18 +185,19 @@ module "bigtable-instance" {
|
|||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [name](variables.tf#L56) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L67) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
|
||||
| [zone](variables.tf#L88) | The zone to create the Cloud Bigtable cluster in. | <code>string</code> | ✓ | |
|
||||
| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | <code title="object({ min_nodes = number max_nodes = number cpu_target = number, storage_target = optional(number, null) })">object({…})</code> | | <code>null</code> |
|
||||
| [name](variables.tf#L69) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L80) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
|
||||
| [zone](variables.tf#L110) | The zone to create the Cloud Bigtable cluster in. | <code>string</code> | ✓ | |
|
||||
| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | <code title="object({ min_nodes = number max_nodes = number cpu_target = number storage_target = optional(number, null) })">object({…})</code> | | <code>null</code> |
|
||||
| [cluster_id](variables.tf#L28) | The ID of the Cloud Bigtable cluster. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [deletion_protection](variables.tf#L34) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
|
||||
| [display_name](variables.tf#L39) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
|
||||
| [iam](variables.tf#L44) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [instance_type](variables.tf#L50) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
|
||||
| [num_nodes](variables.tf#L61) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | <code>number</code> | | <code>1</code> |
|
||||
| [storage_type](variables.tf#L72) | The storage type to use. | <code>string</code> | | <code>"SSD"</code> |
|
||||
| [tables](variables.tf#L78) | Tables to be created in the BigTable instance. | <code title="map(object({ split_keys = optional(list(string), []) column_families = optional(list(string), []) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [default_gc_policy](variables.tf#L34) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||
| [deletion_protection](variables.tf#L46) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
|
||||
| [display_name](variables.tf#L51) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
|
||||
| [iam](variables.tf#L57) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [instance_type](variables.tf#L63) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
|
||||
| [num_nodes](variables.tf#L74) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | <code>number</code> | | <code>1</code> |
|
||||
| [storage_type](variables.tf#L85) | The storage type to use. | <code>string</code> | | <code>"SSD"</code> |
|
||||
| [tables](variables.tf#L91) | Tables to be created in the BigTable instance. | <code title="map(object({ split_keys = optional(list(string), []) column_families = optional(map(object( { gc_policy = optional(object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) }), null) })), {}) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,15 @@
|
|||
|
||||
locals {
|
||||
num_nodes = var.autoscaling_config == null ? var.num_nodes : null
|
||||
gc_pairs = flatten([
|
||||
for table, table_obj in var.tables : [
|
||||
for cf, cf_obj in table_obj.column_families : {
|
||||
table = table
|
||||
column_family = cf
|
||||
gc_policy = cf_obj.gc_policy == null ? var.default_gc_policy : cf_obj.gc_policy
|
||||
}
|
||||
]
|
||||
])
|
||||
}
|
||||
|
||||
resource "google_bigtable_instance" "default" {
|
||||
|
|
@ -61,7 +70,34 @@ resource "google_bigtable_table" "default" {
|
|||
for_each = each.value.column_families
|
||||
|
||||
content {
|
||||
family = column_family.value
|
||||
family = column_family.key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_bigtable_gc_policy" "default" {
|
||||
for_each = { for k, v in local.gc_pairs : k => v if v.gc_policy != null }
|
||||
|
||||
table = each.value.table
|
||||
column_family = each.value.column_family
|
||||
instance_name = google_bigtable_instance.default.name
|
||||
project = var.project_id
|
||||
|
||||
gc_rules = try(each.value.gc_policy.gc_rules, null)
|
||||
mode = try(each.value.gc_policy.mode, null)
|
||||
deletion_policy = try(each.value.gc_policy.deletion_policy, null)
|
||||
|
||||
dynamic "max_age" {
|
||||
for_each = try(each.value.gc_policy.max_age, null) != null ? [""] : []
|
||||
content {
|
||||
duration = each.value.gc_policy.max_age
|
||||
}
|
||||
}
|
||||
|
||||
dynamic "max_version" {
|
||||
for_each = try(each.value.gc_policy.max_version, null) != null ? [""] : []
|
||||
content {
|
||||
number = each.value.gc_policy.max_version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ variable "autoscaling_config" {
|
|||
type = object({
|
||||
min_nodes = number
|
||||
max_nodes = number
|
||||
cpu_target = number,
|
||||
cpu_target = number
|
||||
storage_target = optional(number, null)
|
||||
})
|
||||
default = null
|
||||
|
|
@ -31,6 +31,18 @@ variable "cluster_id" {
|
|||
default = "europe-west1"
|
||||
}
|
||||
|
||||
variable "default_gc_policy" {
|
||||
description = "Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families."
|
||||
type = object({
|
||||
deletion_policy = optional(string)
|
||||
gc_rules = optional(string)
|
||||
mode = optional(string)
|
||||
max_age = optional(string)
|
||||
max_version = optional(string)
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "deletion_protection" {
|
||||
description = "Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail."
|
||||
default = true
|
||||
|
|
@ -41,6 +53,7 @@ variable "display_name" {
|
|||
default = null
|
||||
}
|
||||
|
||||
|
||||
variable "iam" {
|
||||
description = "IAM bindings for topic in {ROLE => [MEMBERS]} format."
|
||||
type = map(list(string))
|
||||
|
|
@ -79,8 +92,17 @@ variable "tables" {
|
|||
description = "Tables to be created in the BigTable instance."
|
||||
nullable = false
|
||||
type = map(object({
|
||||
split_keys = optional(list(string), [])
|
||||
column_families = optional(list(string), [])
|
||||
split_keys = optional(list(string), [])
|
||||
column_families = optional(map(object(
|
||||
{
|
||||
gc_policy = optional(object({
|
||||
deletion_policy = optional(string)
|
||||
gc_rules = optional(string)
|
||||
mode = optional(string)
|
||||
max_age = optional(string)
|
||||
max_version = optional(string)
|
||||
}), null)
|
||||
})), {})
|
||||
}))
|
||||
default = {}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,11 +17,11 @@ terraform {
|
|||
required_providers {
|
||||
google = {
|
||||
source = "hashicorp/google"
|
||||
version = ">= 4.40.0" # tftest
|
||||
version = ">= 4.47.0" # tftest
|
||||
}
|
||||
google-beta = {
|
||||
source = "hashicorp/google-beta"
|
||||
version = ">= 4.40.0" # tftest
|
||||
version = ">= 4.47.0" # tftest
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue