From e3a1a8da4b0ab8d02e6b530df3e9eed31f71fdbb Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 3 May 2020 15:10:36 +0200 Subject: [PATCH] change access variables in bigquery-dataset to support dyamic values --- modules/bigquery-dataset/README.md | 33 ++++--- modules/bigquery-dataset/main.tf | 121 ++++++++++++++++---------- modules/bigquery-dataset/outputs.tf | 35 +++++++- modules/bigquery-dataset/variables.tf | 36 ++++---- 4 files changed, 149 insertions(+), 76 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index e14fbca4..ecbbe8f1 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -4,7 +4,6 @@ This module allows managing a single BigQuery dataset, including access configur ## TODO -- [ ] add support for dynamic values in access identities - [ ] check for dynamic values in tables and views - [ ] add support for external tables @@ -12,17 +11,24 @@ This module allows managing a single BigQuery dataset, including access configur ### Simple dataset with access configuration -Access configuration defaults to using incremental accesses, which add to the default ones set at dataset creation. You can use the `access_authoritative` variable to switch to authoritative mode and have full control over dataset-level access. Be sure to always have at least one `OWNER` access and to avoid duplicating accesses, or `terraform apply` will fail. +Access configuration defaults to using the separate `google_bigquery_dataset_access` resource, so as to leave the default dataset access rules untouched. + +You can choose to manage the `google_bigquery_dataset` access rules instead via the `dataset_access` variable, but be sure to always have at least one `OWNER` access and to avoid duplicating accesses, or `terraform apply` will fail. + +The access variables are split into `access_roles` and `access_identities` variables, so that dynamic values can be passed in for identities (eg a service account email generated by a different module or resource). The `access_views` variable is separate, so as to allow proper type constraints. ```hcl module "bigquery-dataset" { source = "./modules/bigquery-dataset" project_id = "my-project - id = "my-dataset" - access = { - "OWNER" = [ - { identity_type = "group_by_email", identity = "dataset-owners@example.com" } - ] + id = "my-dataset" + access_roles = { + reader-group = { role = "READER", type = "group_by_email" } + owner = { role = "OWNER", type = "user_by_email" } + } + access_identities = { + reader-group = "playground-test@ludomagno.net" + owner = "ludo@ludomagno.net" } } ``` @@ -126,9 +132,10 @@ module "bigquery-dataset" { |---|---|:---: |:---:|:---:| | id | Dataset id. | string | ✓ | | | project_id | Id of the project where datasets will be created. | string | ✓ | | -| *access* | Dataset access rules keyed by role, valid identity types are `domain`, `group_by_email`, `special_group` and `user_by_email`. Mode can be controlled via the `access_authoritative` variable. | map(list(object({...}))) | | {} | -| *access_authoritative* | Use authoritative access instead of additive. | bool | | false | -| *access_views* | Dataset access rules for views. Mode can be controlled via the `access_authoritative` variable. | list(object({...})) | | [] | +| *access_identities* | Map of access identities used for access roles with type different from `view`. A separate variable is needed as identities can be set to dynamic values. | map(string) | | {} | +| *access_roles* | Map of access rules with role and identity type. Keys are arbitrary and only used to combine identities with each role. Valid types are `domain`, `group_by_email`, `special_group`, `user_by_email`, `view`. | map(object({...})) | | {} | +| *access_views* | Map of view data for access roles with identity type equal to `view`. A separate variable is needed as identities can be set to dynamic values. | map(object({...})) | | {} | +| *dataset_access* | Set access in the dataset resource instead of using separate resources. | bool | | false | | *encryption_key* | Self link of the KMS key that will be used to protect destination table. | string | | null | | *friendly_name* | Dataset friendly name. | string | | null | | *labels* | Dataset labels. | map(string) | | {} | @@ -142,10 +149,12 @@ module "bigquery-dataset" { | name | description | sensitive | |---|---|:---:| | dataset | Dataset resource. | | -| dataset_id | Dataset full id. | | -| id | Dataset id. | | +| dataset_id | Dataset id. | | +| id | Fully qualified dataset id. | | | self_link | Dataset self link. | | +| table_ids | Map of fully qualified table ids keyed by table ids. | | | tables | Table resources. | | +| view_ids | Map of fully qualified view ids keyed by view ids. | | | views | View resources. | | diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index 2e8f6cec..4df526f7 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -15,18 +15,20 @@ */ locals { - access_list = flatten([ - for role, accesses in var.access : [ - for access in accesses : merge(access, { role = role }) - ] - ]) - access_default = { - for access in local.access_list : - "${access.role}-${access.identity_type}-${access.identity}" => access + access_domain = { + for k, v in var.access_roles : k => v if v.type == "domain" } - access_views = { - for access in var.access_views : - "${access.project_id}-${access.dataset_id}-${access.table_id}" => access + access_group = { + for k, v in var.access_roles : k => v if v.type == "group_by_email" + } + access_special = { + for k, v in var.access_roles : k => v if v.type == "special_group" + } + access_user = { + for k, v in var.access_roles : k => v if v.type == "user_by_email" + } + access_view = { + for k, v in var.access_roles : k => v if v.type == "view" } } @@ -43,31 +45,44 @@ resource "google_bigquery_dataset" "default" { default_partition_expiration_ms = var.options.default_partition_expiration_ms dynamic access { - for_each = var.access_authoritative ? local.access_default : {} + for_each = var.dataset_access ? local.access_domain : {} content { - role = access.value.role - domain = ( - access.value.identity_type == "domain" ? access.value.identity : null - ) - group_by_email = ( - access.value.identity_type == "group_by_email" ? access.value.identity : null - ) - special_group = ( - access.value.identity_type == "special_group" ? access.value.identity : null - ) - user_by_email = ( - access.value.identity_type == "user_by_email" ? access.value.identity : null - ) + role = access.value.role + domain = try(var.access_identities[access.key]) } } dynamic access { - for_each = var.access_authoritative ? local.access_views : {} + for_each = var.dataset_access ? local.access_group : {} + content { + role = access.value.role + group_by_email = try(var.access_identities[access.key]) + } + } + + dynamic access { + for_each = var.dataset_access ? local.access_special : {} + content { + role = access.value.role + special_group = try(var.access_identities[access.key]) + } + } + + dynamic access { + for_each = var.dataset_access ? local.access_user : {} + content { + role = access.value.role + user_by_email = try(var.access_identities[access.key]) + } + } + + dynamic access { + for_each = var.dataset_access ? local.access_view : {} content { view { - project_id = access.value.project_id - dataset_id = access.value.dataset_id - table_id = access.value.table_id + project_id = try(var.access.views[access.key].project_id, null) + dataset_id = try(var.access.views[access.key].dataset_id, null) + table_id = try(var.access.views[access.key].table_id, null) } } } @@ -81,34 +96,46 @@ resource "google_bigquery_dataset" "default" { } -resource "google_bigquery_dataset_access" "default" { - for_each = var.access_authoritative ? {} : local.access_default +resource "google_bigquery_dataset_access" "domain" { + for_each = var.dataset_access ? {} : local.access_domain project = var.project_id dataset_id = google_bigquery_dataset.default.dataset_id role = each.value.role - domain = ( - each.value.identity_type == "domain" ? each.value.identity : null - ) - group_by_email = ( - each.value.identity_type == "group_by_email" ? each.value.identity : null - ) - special_group = ( - each.value.identity_type == "special_group" ? each.value.identity : null - ) - user_by_email = ( - each.value.identity_type == "user_by_email" ? each.value.identity : null - ) + domain = try(var.access_identities[each.key]) } +resource "google_bigquery_dataset_access" "group_by_email" { + for_each = var.dataset_access ? {} : local.access_group + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = each.value.role + group_by_email = try(var.access_identities[each.key]) +} + +resource "google_bigquery_dataset_access" "special_group" { + for_each = var.dataset_access ? {} : local.access_special + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = each.value.role + special_group = try(var.access_identities[each.key]) +} + +resource "google_bigquery_dataset_access" "user_by_email" { + for_each = var.dataset_access ? {} : local.access_user + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = each.value.role + user_by_email = try(var.access_identities[each.key]) +} resource "google_bigquery_dataset_access" "views" { - for_each = var.access_authoritative ? {} : local.access_views + for_each = var.dataset_access ? {} : local.access_view project = var.project_id dataset_id = google_bigquery_dataset.default.dataset_id view { - project_id = each.value.project - dataset_id = each.value.dataset_id - table_id = each.value.table_id + project_id = try(var.access_views[each.key].project_id, null) + dataset_id = try(var.access_views[each.key].dataset_id, null) + table_id = try(var.access_views[each.key].table_id, null) } } diff --git a/modules/bigquery-dataset/outputs.tf b/modules/bigquery-dataset/outputs.tf index ede802fb..44753fad 100644 --- a/modules/bigquery-dataset/outputs.tf +++ b/modules/bigquery-dataset/outputs.tf @@ -20,18 +20,39 @@ output "dataset" { } output "dataset_id" { - description = "Dataset full id." + description = "Dataset id." value = google_bigquery_dataset.default.dataset_id + depends_on = [ + google_bigquery_dataset_access.domain, + google_bigquery_dataset_access.group_by_email, + google_bigquery_dataset_access.special_group, + google_bigquery_dataset_access.user_by_email, + google_bigquery_dataset_access.views + ] } output "id" { - description = "Dataset id." + description = "Fully qualified dataset id." value = google_bigquery_dataset.default.id + depends_on = [ + google_bigquery_dataset_access.domain, + google_bigquery_dataset_access.group_by_email, + google_bigquery_dataset_access.special_group, + google_bigquery_dataset_access.user_by_email, + google_bigquery_dataset_access.views + ] } output "self_link" { description = "Dataset self link." value = google_bigquery_dataset.default.self_link + depends_on = [ + google_bigquery_dataset_access.domain, + google_bigquery_dataset_access.group_by_email, + google_bigquery_dataset_access.special_group, + google_bigquery_dataset_access.user_by_email, + google_bigquery_dataset_access.views + ] } output "tables" { @@ -39,7 +60,17 @@ output "tables" { value = google_bigquery_table.default } +output "table_ids" { + description = "Map of fully qualified table ids keyed by table ids." + value = { for k, v in google_bigquery_table.default : v.table_id => v.id } +} + output "views" { description = "View resources." value = google_bigquery_table.views } + +output "view_ids" { + description = "Map of fully qualified view ids keyed by view ids." + value = { for k, v in google_bigquery_table.views : v.table_id => v.id } +} diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index fca9378f..54555b54 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -14,29 +14,35 @@ * limitations under the License. */ -variable "access" { - description = "Dataset access rules keyed by role, valid identity types are `domain`, `group_by_email`, `special_group` and `user_by_email`. Mode can be controlled via the `access_authoritative` variable." - type = map(list(object({ - identity_type = string - identity = string - }))) +variable "access_identities" { + description = "Map of access identities used for access roles with type different from `view`. A separate variable is needed as identities can be set to dynamic values." + type = map(string) + default = {} +} + +variable "access_roles" { + description = "Map of access rules with role and identity type. Keys are arbitrary and only used to combine identities with each role. Valid types are `domain`, `group_by_email`, `special_group`, `user_by_email`, `view`." + type = map(object({ + role = string + type = string + })) default = {} } -variable "access_authoritative" { - description = "Use authoritative access instead of additive." - type = bool - default = false -} - variable "access_views" { - description = "Dataset access rules for views. Mode can be controlled via the `access_authoritative` variable." - type = list(object({ + description = "Map of view data for access roles with identity type equal to `view`. A separate variable is needed as identities can be set to dynamic values." + type = map(object({ project_id = string dataset_id = string table_id = string })) - default = [] + default = {} +} + +variable "dataset_access" { + description = "Set access in the dataset resource instead of using separate resources." + type = bool + default = false } variable "encryption_key" {