Add gc_policy to Bigtable module.
Column families have now a new property, to specify the garbage collection policy. A new option also allows to set a default policy if none is specified. This changes the previous syntax for column families, that was helpful since the policy is column-family specific, the new syntax makes it easier to specify a policy per column family.
This commit is contained in:
parent
008828f2d8
commit
32bee7104e
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.40.0" # tftest
|
version = ">= 4.47.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.40.0" # tftest
|
version = ">= 4.47.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@ This module allows managing a single BigTable instance, including access configu
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
- [ ] support bigtable_gc_policy
|
|
||||||
- [ ] support bigtable_app_profile
|
- [ ] support bigtable_app_profile
|
||||||
- [ ] support cluster replicas
|
- [ ] support cluster replicas
|
||||||
- [ ] support IAM for tables
|
- [ ] support IAM for tables
|
||||||
|
@ -48,10 +47,74 @@ module "bigtable-instance" {
|
||||||
test1 = {},
|
test1 = {},
|
||||||
test2 = {
|
test2 = {
|
||||||
split_keys = ["a", "b", "c"]
|
split_keys = ["a", "b", "c"]
|
||||||
column_families = ["cf1", "cf2", "cf3"]
|
column_families = {
|
||||||
|
cf1 = {}
|
||||||
|
cf2 = {}
|
||||||
|
cf3 = {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
test3 = {
|
test3 = {
|
||||||
column_families = ["cf1"]
|
column_families = {
|
||||||
|
cf1 = {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# tftest modules=1 resources=4
|
||||||
|
```
|
||||||
|
|
||||||
|
### Instance with garbage collection policy
|
||||||
|
|
||||||
|
```hcl
|
||||||
|
|
||||||
|
module "bigtable-instance" {
|
||||||
|
source = "./fabric/modules/bigtable-instance"
|
||||||
|
project_id = "my-project"
|
||||||
|
name = "instance"
|
||||||
|
cluster_id = "instance"
|
||||||
|
zone = "europe-west1-b"
|
||||||
|
tables = {
|
||||||
|
test1 = {
|
||||||
|
column_families = {
|
||||||
|
cf1 = {
|
||||||
|
gc_policy = {
|
||||||
|
deletion_policy = "ABANDON"
|
||||||
|
max_age = "18h"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cf2 = {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# tftest modules=1 resources=3
|
||||||
|
```
|
||||||
|
|
||||||
|
### Instance with default garbage collection policy
|
||||||
|
|
||||||
|
The default garbage collection policy is applied to any column family that does
|
||||||
|
not specify a `gc_policy`. If a column family specifies a `gc_policy`, the
|
||||||
|
default garbage collection policy is ignored for that column family.
|
||||||
|
|
||||||
|
```hcl
|
||||||
|
|
||||||
|
module "bigtable-instance" {
|
||||||
|
source = "./fabric/modules/bigtable-instance"
|
||||||
|
project_id = "my-project"
|
||||||
|
name = "instance"
|
||||||
|
cluster_id = "instance"
|
||||||
|
zone = "europe-west1-b"
|
||||||
|
default_gc_policy = {
|
||||||
|
deletion_policy = "ABANDON"
|
||||||
|
max_age = "18h"
|
||||||
|
max_version = 7
|
||||||
|
}
|
||||||
|
tables = {
|
||||||
|
test1 = {
|
||||||
|
column_families = {
|
||||||
|
cf1 = {}
|
||||||
|
cf2 = {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -122,18 +185,19 @@ module "bigtable-instance" {
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [name](variables.tf#L56) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
|
| [name](variables.tf#L69) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ | |
|
||||||
| [project_id](variables.tf#L67) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
|
| [project_id](variables.tf#L80) | Id of the project where datasets will be created. | <code>string</code> | ✓ | |
|
||||||
| [zone](variables.tf#L88) | The zone to create the Cloud Bigtable cluster in. | <code>string</code> | ✓ | |
|
| [zone](variables.tf#L110) | The zone to create the Cloud Bigtable cluster in. | <code>string</code> | ✓ | |
|
||||||
| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | <code title="object({ min_nodes = number max_nodes = number cpu_target = number, storage_target = optional(number, null) })">object({…})</code> | | <code>null</code> |
|
| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | <code title="object({ min_nodes = number max_nodes = number cpu_target = number storage_target = optional(number, null) })">object({…})</code> | | <code>null</code> |
|
||||||
| [cluster_id](variables.tf#L28) | The ID of the Cloud Bigtable cluster. | <code>string</code> | | <code>"europe-west1"</code> |
|
| [cluster_id](variables.tf#L28) | The ID of the Cloud Bigtable cluster. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||||
| [deletion_protection](variables.tf#L34) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
|
| [default_gc_policy](variables.tf#L34) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||||
| [display_name](variables.tf#L39) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
|
| [deletion_protection](variables.tf#L46) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> | | <code>true</code> |
|
||||||
| [iam](variables.tf#L44) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
| [display_name](variables.tf#L51) | The human-readable display name of the Bigtable instance. | <code></code> | | <code>null</code> |
|
||||||
| [instance_type](variables.tf#L50) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
|
| [iam](variables.tf#L57) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||||
| [num_nodes](variables.tf#L61) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | <code>number</code> | | <code>1</code> |
|
| [instance_type](variables.tf#L63) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> | | <code>null</code> |
|
||||||
| [storage_type](variables.tf#L72) | The storage type to use. | <code>string</code> | | <code>"SSD"</code> |
|
| [num_nodes](variables.tf#L74) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | <code>number</code> | | <code>1</code> |
|
||||||
| [tables](variables.tf#L78) | Tables to be created in the BigTable instance. | <code title="map(object({ split_keys = optional(list(string), []) column_families = optional(list(string), []) }))">map(object({…}))</code> | | <code>{}</code> |
|
| [storage_type](variables.tf#L85) | The storage type to use. | <code>string</code> | | <code>"SSD"</code> |
|
||||||
|
| [tables](variables.tf#L91) | Tables to be created in the BigTable instance. | <code title="map(object({ split_keys = optional(list(string), []) column_families = optional(map(object( { gc_policy = optional(object({ deletion_policy = optional(string) gc_rules = optional(string) mode = optional(string) max_age = optional(string) max_version = optional(string) }), null) })), {}) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
|
|
|
@ -16,6 +16,15 @@
|
||||||
|
|
||||||
locals {
|
locals {
|
||||||
num_nodes = var.autoscaling_config == null ? var.num_nodes : null
|
num_nodes = var.autoscaling_config == null ? var.num_nodes : null
|
||||||
|
gc_pairs = flatten([
|
||||||
|
for table, table_obj in var.tables : [
|
||||||
|
for cf, cf_obj in table_obj.column_families : {
|
||||||
|
table = table
|
||||||
|
column_family = cf
|
||||||
|
gc_policy = cf_obj.gc_policy == null ? var.default_gc_policy : cf_obj.gc_policy
|
||||||
|
}
|
||||||
|
]
|
||||||
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_bigtable_instance" "default" {
|
resource "google_bigtable_instance" "default" {
|
||||||
|
@ -61,7 +70,34 @@ resource "google_bigtable_table" "default" {
|
||||||
for_each = each.value.column_families
|
for_each = each.value.column_families
|
||||||
|
|
||||||
content {
|
content {
|
||||||
family = column_family.value
|
family = column_family.key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "google_bigtable_gc_policy" "default" {
|
||||||
|
for_each = { for k, v in local.gc_pairs : k => v if v.gc_policy != null }
|
||||||
|
|
||||||
|
table = each.value.table
|
||||||
|
column_family = each.value.column_family
|
||||||
|
instance_name = google_bigtable_instance.default.name
|
||||||
|
project = var.project_id
|
||||||
|
|
||||||
|
gc_rules = try(each.value.gc_policy.gc_rules, null)
|
||||||
|
mode = try(each.value.gc_policy.mode, null)
|
||||||
|
deletion_policy = try(each.value.gc_policy.deletion_policy, null)
|
||||||
|
|
||||||
|
dynamic "max_age" {
|
||||||
|
for_each = try(each.value.gc_policy.max_age, null) != null ? [""] : []
|
||||||
|
content {
|
||||||
|
duration = each.value.gc_policy.max_age
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dynamic "max_version" {
|
||||||
|
for_each = try(each.value.gc_policy.max_version, null) != null ? [""] : []
|
||||||
|
content {
|
||||||
|
number = each.value.gc_policy.max_version
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,7 @@ variable "autoscaling_config" {
|
||||||
type = object({
|
type = object({
|
||||||
min_nodes = number
|
min_nodes = number
|
||||||
max_nodes = number
|
max_nodes = number
|
||||||
cpu_target = number,
|
cpu_target = number
|
||||||
storage_target = optional(number, null)
|
storage_target = optional(number, null)
|
||||||
})
|
})
|
||||||
default = null
|
default = null
|
||||||
|
@ -31,6 +31,18 @@ variable "cluster_id" {
|
||||||
default = "europe-west1"
|
default = "europe-west1"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "default_gc_policy" {
|
||||||
|
description = "Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families."
|
||||||
|
type = object({
|
||||||
|
deletion_policy = optional(string)
|
||||||
|
gc_rules = optional(string)
|
||||||
|
mode = optional(string)
|
||||||
|
max_age = optional(string)
|
||||||
|
max_version = optional(string)
|
||||||
|
})
|
||||||
|
default = null
|
||||||
|
}
|
||||||
|
|
||||||
variable "deletion_protection" {
|
variable "deletion_protection" {
|
||||||
description = "Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail."
|
description = "Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail."
|
||||||
default = true
|
default = true
|
||||||
|
@ -41,6 +53,7 @@ variable "display_name" {
|
||||||
default = null
|
default = null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
variable "iam" {
|
variable "iam" {
|
||||||
description = "IAM bindings for topic in {ROLE => [MEMBERS]} format."
|
description = "IAM bindings for topic in {ROLE => [MEMBERS]} format."
|
||||||
type = map(list(string))
|
type = map(list(string))
|
||||||
|
@ -80,7 +93,16 @@ variable "tables" {
|
||||||
nullable = false
|
nullable = false
|
||||||
type = map(object({
|
type = map(object({
|
||||||
split_keys = optional(list(string), [])
|
split_keys = optional(list(string), [])
|
||||||
column_families = optional(list(string), [])
|
column_families = optional(map(object(
|
||||||
|
{
|
||||||
|
gc_policy = optional(object({
|
||||||
|
deletion_policy = optional(string)
|
||||||
|
gc_rules = optional(string)
|
||||||
|
mode = optional(string)
|
||||||
|
max_age = optional(string)
|
||||||
|
max_version = optional(string)
|
||||||
|
}), null)
|
||||||
|
})), {})
|
||||||
}))
|
}))
|
||||||
default = {}
|
default = {}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,11 +17,11 @@ terraform {
|
||||||
required_providers {
|
required_providers {
|
||||||
google = {
|
google = {
|
||||||
source = "hashicorp/google"
|
source = "hashicorp/google"
|
||||||
version = ">= 4.40.0" # tftest
|
version = ">= 4.47.0" # tftest
|
||||||
}
|
}
|
||||||
google-beta = {
|
google-beta = {
|
||||||
source = "hashicorp/google-beta"
|
source = "hashicorp/google-beta"
|
||||||
version = ">= 4.40.0" # tftest
|
version = ">= 4.47.0" # tftest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue