From 9706d2c386be8106e761e5de13a209e48ce76616 Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Tue, 9 Jun 2020 02:40:26 +0200 Subject: [PATCH 1/4] Add BigTable module --- modules/bigtable-instance/README.md | 59 ++++++++++++++ modules/bigtable-instance/main.tf | 68 ++++++++++++++++ modules/bigtable-instance/outputs.tf | 46 +++++++++++ modules/bigtable-instance/variables.tf | 105 +++++++++++++++++++++++++ modules/bigtable-instance/versions.tf | 19 +++++ 5 files changed, 297 insertions(+) create mode 100644 modules/bigtable-instance/README.md create mode 100644 modules/bigtable-instance/main.tf create mode 100644 modules/bigtable-instance/outputs.tf create mode 100644 modules/bigtable-instance/variables.tf create mode 100644 modules/bigtable-instance/versions.tf diff --git a/modules/bigtable-instance/README.md b/modules/bigtable-instance/README.md new file mode 100644 index 00000000..d34e6bbc --- /dev/null +++ b/modules/bigtable-instance/README.md @@ -0,0 +1,59 @@ +# Google Cloud BigTable Module + +This module allows managing a single BigTable instance, including access configuration and tables. + +## TODO + +- [ ] support bigtable_gc_policy +- [ ] support bigtable_app_profile + +## Examples + +### Simple instance with access configuration + +```hcl + +module "big-table-instance" { + source = "./modules/bigtable-instance" + project_id = "my-project" + name = "instance" + cluster_id = "instance" + instance_type = "PRODUCTION" + tables = ["table1","table2"] + access_roles = ["viewer"] + access_roles_binding = { + viewer = ["user:viewer@testdomain.com"] + } +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | he name of the Cloud Bigtable instance. | string | ✓ | | +| project_id | Id of the project where datasets will be created. | string | ✓ | | +| *access_roles* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. | list(string) | | [] | +| *access_roles_binding* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved. | map(list(string)) | | {} | +| *cluster_id* | The ID of the Cloud Bigtable cluster. | string | | europe-west1 | +| *deletion_protection* | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | | | true | +| *display_name* | The human-readable display name of the Bigtable instance. | | | null | +| *instance_type* | None | string | | DEVELOPMENT | +| *num_nodes* | The number of nodes in your Cloud Bigtable cluster. | number | | 1 | +| *storage_type* | The storage type to use. | string | | SSD | +| *table_options_default* | Default option of tables created in the BigTable instnace. | object({...}) | | ... | +| *tables* | Tables to be created in the BigTable instnace. | list(string) | | [] | +| *tables_options* | Tables to be created in the BigTable instnace. | map(object({...})) | | {} | +| *zone* | The zone to create the Cloud Bigtable cluster in. | string | | europe-west1-b | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | An identifier for the resource with format projects/{{project}}/instances/{{name}}. | | +| instance | BigTable intance. | | +| table_ids | Map of fully qualified table ids keyed by table name. | | +| tables | Table resources. | | + + diff --git a/modules/bigtable-instance/main.tf b/modules/bigtable-instance/main.tf new file mode 100644 index 00000000..e8beacf8 --- /dev/null +++ b/modules/bigtable-instance/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + tables = { + for k in var.tables : k => lookup(var.tables_options, k, var.table_options_default) + } + + access_roles_bindings = { + for k in var.access_roles : k => lookup(var.access_roles_binding, k, []) + } +} + +resource "google_bigtable_instance" "default" { + project = var.project_id + name = var.name + cluster { + cluster_id = var.cluster_id + zone = var.zone + storage_type = var.storage_type + } + instance_type = var.instance_type + + display_name = var.display_name == null ? var.display_name : var.name + deletion_protection = var.deletion_protection +} + +resource "google_bigtable_instance_iam_binding" "default" { + for_each = local.access_roles_bindings + + project = var.project_id + instance = google_bigtable_instance.default.name + role = "roles/bigtable.${each.key}" + members = each.value +} + +resource "google_bigtable_table" "default" { + for_each = local.tables + project = var.project_id + instance_name = google_bigtable_instance.default.name + name = each.key + split_keys = each.value.split_keys + + dynamic column_family { + for_each = each.value.column_family != null ? [""] : [] + + content { + family = each.value.column_family + } + } + + # lifecycle { + # prevent_destroy = true + # } +} diff --git a/modules/bigtable-instance/outputs.tf b/modules/bigtable-instance/outputs.tf new file mode 100644 index 00000000..2012b5c6 --- /dev/null +++ b/modules/bigtable-instance/outputs.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "An identifier for the resource with format projects/{{project}}/instances/{{name}}." + value = google_bigtable_instance.default.id + depends_on = [ + google_bigtable_instance_iam_binding, + google_bigtable_table + ] +} + +output "instance" { + description = "BigTable intance." + value = google_bigtable_instance.default + depends_on = [ + google_bigtable_instance_iam_binding, + google_bigtable_table + ] +} + +output "tables" { + description = "Table resources." + value = google_bigtable_table.default +} + +output "table_ids" { + description = "Map of fully qualified table ids keyed by table name." + value = { for k, v in google_bigtable_table.default : v.name => v.id } +} + + + diff --git a/modules/bigtable-instance/variables.tf b/modules/bigtable-instance/variables.tf new file mode 100644 index 00000000..16066b13 --- /dev/null +++ b/modules/bigtable-instance/variables.tf @@ -0,0 +1,105 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "access_roles" { + description = "Authoritative for a given role. Updates the IAM policy to grant a role to a list of members." + type = list(string) + default = [] +} + +variable "access_roles_binding" { + description = "Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved." + type = map(list(string)) + default = {} +} + +variable "cluster_id" { + description = "The ID of the Cloud Bigtable cluster." + type = string + default = "europe-west1" +} + +variable "deletion_protection" { + description = "Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail." + default = true +} + +variable "display_name" { + description = "The human-readable display name of the Bigtable instance." + default = null +} + +variable "instance_type" { + description = "The instance type to create. One of \"DEVELOPMENT\" or \"PRODUCTION\". Defaults to \"DEVELOPMENT\"" + type = string + default = "DEVELOPMENT" +} + +variable "name" { + description = "he name of the Cloud Bigtable instance." + type = string +} + +variable "num_nodes" { + description = "The number of nodes in your Cloud Bigtable cluster." + type = number + default = 1 +} + +variable "project_id" { + description = "Id of the project where datasets will be created." + type = string +} + +variable "storage_type" { + description = "The storage type to use." + type = string + default = "SSD" +} + +variable "tables" { + description = "Tables to be created in the BigTable instnace." + type = list(string) + default = [] +} + +variable "tables_options" { + description = "Tables to be created in the BigTable instnace." + type = map(object({ + split_keys = list(string) + column_family = string + }) + ) + default = {} +} + +variable "table_options_default" { + description = "Default option of tables created in the BigTable instnace." + type = object({ + split_keys = list(string) + column_family = string + }) + default = { + split_keys = [] + column_family = null + } +} + +variable "zone" { + description = "The zone to create the Cloud Bigtable cluster in." + type = string + default = "europe-west1-b" +} diff --git a/modules/bigtable-instance/versions.tf b/modules/bigtable-instance/versions.tf new file mode 100644 index 00000000..ce6918e0 --- /dev/null +++ b/modules/bigtable-instance/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} From 8d75a278a62c88eabc19c1f3cbb4c99b08511bef Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Tue, 9 Jun 2020 15:55:22 +0200 Subject: [PATCH 2/4] Fixes --- modules/bigtable-instance/main.tf | 8 ++++---- modules/bigtable-instance/variables.tf | 26 ++++++++++---------------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/modules/bigtable-instance/main.tf b/modules/bigtable-instance/main.tf index e8beacf8..0e7129ff 100644 --- a/modules/bigtable-instance/main.tf +++ b/modules/bigtable-instance/main.tf @@ -16,11 +16,11 @@ locals { tables = { - for k in var.tables : k => lookup(var.tables_options, k, var.table_options_default) + for k, v in var.tables : k => v.table_options != null ? v.table_options : var.table_options_defaults } - access_roles_bindings = { - for k in var.access_roles : k => lookup(var.access_roles_binding, k, []) + iam_roles_bindings = { + for k in var.iam_roles : k => lookup(var.iam_members, k, []) } } @@ -39,7 +39,7 @@ resource "google_bigtable_instance" "default" { } resource "google_bigtable_instance_iam_binding" "default" { - for_each = local.access_roles_bindings + for_each = local.iam_roles_bindings project = var.project_id instance = google_bigtable_instance.default.name diff --git a/modules/bigtable-instance/variables.tf b/modules/bigtable-instance/variables.tf index 16066b13..982bfb8a 100644 --- a/modules/bigtable-instance/variables.tf +++ b/modules/bigtable-instance/variables.tf @@ -14,13 +14,13 @@ * limitations under the License. */ -variable "access_roles" { +variable "iam_roles" { description = "Authoritative for a given role. Updates the IAM policy to grant a role to a list of members." type = list(string) default = [] } -variable "access_roles_binding" { +variable "iam_members" { description = "Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved." type = map(list(string)) default = {} @@ -49,7 +49,7 @@ variable "instance_type" { } variable "name" { - description = "he name of the Cloud Bigtable instance." + description = "The name of the Cloud Bigtable instance." type = string } @@ -71,23 +71,18 @@ variable "storage_type" { } variable "tables" { - description = "Tables to be created in the BigTable instnace." - type = list(string) - default = [] -} - -variable "tables_options" { - description = "Tables to be created in the BigTable instnace." + description = "Tables to be created in the BigTable instance." type = map(object({ - split_keys = list(string) - column_family = string + table_options = object({ + split_keys = list(string) + column_family = string }) - ) + })) default = {} } -variable "table_options_default" { - description = "Default option of tables created in the BigTable instnace." +variable "table_options_defaults" { + description = "Default option of tables created in the BigTable instance." type = object({ split_keys = list(string) column_family = string @@ -101,5 +96,4 @@ variable "table_options_default" { variable "zone" { description = "The zone to create the Cloud Bigtable cluster in." type = string - default = "europe-west1-b" } From 2f97d246f60bd3f5f1a91add3ff4c80778175da9 Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Tue, 9 Jun 2020 16:03:10 +0200 Subject: [PATCH 3/4] Fix README --- modules/bigtable-instance/README.md | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/modules/bigtable-instance/README.md b/modules/bigtable-instance/README.md index d34e6bbc..f63e1cf0 100644 --- a/modules/bigtable-instance/README.md +++ b/modules/bigtable-instance/README.md @@ -19,9 +19,16 @@ module "big-table-instance" { name = "instance" cluster_id = "instance" instance_type = "PRODUCTION" - tables = ["table1","table2"] - access_roles = ["viewer"] - access_roles_binding = { + tables = { + test1 = { table_options = null }, + test2 = { table_options = { + split_keys = ["a", "b", "c"] + column_family = null + } + } + } + iam_roles = ["viewer"] + iam_members = { viewer = ["user:viewer@testdomain.com"] } } @@ -32,20 +39,19 @@ module "big-table-instance" { | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| name | he name of the Cloud Bigtable instance. | string | ✓ | | +| name | The name of the Cloud Bigtable instance. | string | ✓ | | | project_id | Id of the project where datasets will be created. | string | ✓ | | -| *access_roles* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. | list(string) | | [] | -| *access_roles_binding* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved. | map(list(string)) | | {} | +| zone | The zone to create the Cloud Bigtable cluster in. | string | ✓ | | | *cluster_id* | The ID of the Cloud Bigtable cluster. | string | | europe-west1 | | *deletion_protection* | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | | | true | | *display_name* | The human-readable display name of the Bigtable instance. | | | null | +| *iam_members* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved. | map(list(string)) | | {} | +| *iam_roles* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. | list(string) | | [] | | *instance_type* | None | string | | DEVELOPMENT | | *num_nodes* | The number of nodes in your Cloud Bigtable cluster. | number | | 1 | | *storage_type* | The storage type to use. | string | | SSD | -| *table_options_default* | Default option of tables created in the BigTable instnace. | object({...}) | | ... | -| *tables* | Tables to be created in the BigTable instnace. | list(string) | | [] | -| *tables_options* | Tables to be created in the BigTable instnace. | map(object({...})) | | {} | -| *zone* | The zone to create the Cloud Bigtable cluster in. | string | | europe-west1-b | +| *table_options_defaults* | Default option of tables created in the BigTable instance. | object({...}) | | ... | +| *tables* | Tables to be created in the BigTable instance. | map(object({...})) | | {} | ## Outputs From 14841180496cde41adcd84e95845deab953c008c Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Tue, 9 Jun 2020 16:25:05 +0200 Subject: [PATCH 4/4] update top-level README update modules/README update CHANGELOG --- CHANGELOG.md | 1 + README.md | 2 +- modules/README.md | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae000b79..806597ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- new `bigtable-instance` module ## [1.8.1] - 2020-06-07 diff --git a/README.md b/README.md index 53aef3c6..71d5f86f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Currently available modules: - **foundational** - [folders](./modules/folders), [log sinks](./modules/logging-sinks), [organization](./modules/organization), [project](./modules/project), [service accounts](./modules/iam-service-accounts) - **networking** - [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN static](./modules/net-vpn-static), [VPN dynamic](./modules/net-vpn-dynamic), [VPN HA](./modules/net-vpn-ha), [NAT](./modules/net-cloudnat), [address reservation](./modules/net-address), [DNS](./modules/dns), [L4 ILB](./modules/net-ilb), [Service Directory](./modules/service-directory) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [GKE cluster](./modules/gke-cluster), [GKE nodepool](./modules/gke-nodepool), [COS container](./modules/cos-container) (coredns, mysql, onprem, squid) -- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion) +- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion), [Bigtable instance](./modules/bigtable-instance) - **security** - [KMS](./modules/kms), [SecretManager](./modules/secret-manager) - **development** - [Cloud Source Repository](./modules/source-repository), [Container Registry](./modules/container-registry), [Artifact Registry](./modules/artifact-registry) diff --git a/modules/README.md b/modules/README.md index b95c87d4..a831c734 100644 --- a/modules/README.md +++ b/modules/README.md @@ -45,6 +45,7 @@ Specific modules also offer support for non-authoritative bindings (e.g. `google - [Datafusion](./datafusion) - [GCS](./gcs) - [Pub/Sub](./pubsub) +- [Bigtable instance](./bigtable-instance) ## Development