diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index a1b2b2dd..b440859b 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -54,6 +54,119 @@ module "bigquery-dataset" { # tftest modules=1 resources=2 inventory=iam.yaml ``` +### Authorized Views, Datasets, and Routines + +You can specify authorized [views](https://cloud.google.com/bigquery/docs/authorized-views), [datasets](https://cloud.google.com/bigquery/docs/authorized-datasets?hl=en), and [routines](https://cloud.google.com/bigquery/docs/authorized-routines) via the `authorized_views`, `authorized_datasets` and `authorized_routines` variables, respectively. + +```hcl +// Create private BigQuery dataset that will not be publicly accessible, except via the authorized BigQuery resources +module "bigquery-dataset-private" { + source = "./fabric/modules/bigquery-dataset" + project_id = "private_project" + id = "private_dataset" + authorized_views = [ + { + project_id = "auth_view_project" + dataset_id = "auth_view_dataset" + table_id = "auth_view" + } + ] + authorized_datasets = [ + { + project_id = "auth_dataset_project" + dataset_id = "auth_dataset" + } + ] + authorized_routines = [ + { + project_id = "auth_routine_project" + dataset_id = "auth_routine_dataset" + routine_id = "auth_routine" + } + ] +} + +// Create authorized view in a public dataset +module "bigquery-authorized-views-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_view_project" + id = "auth_view_dataset" + views = { + auth_view = { + friendly_name = "Public" + labels = {} + query = "SELECT * FROM `private_project.private_dataset.private_table`" + use_legacy_sql = false + deletion_protection = true + } + } +} + +// Create public authorized dataset +module "bigquery-authorized-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_dataset_project" + id = "auth_dataset" +} + +// Create public authorized routine +module "bigquery-authorized-authorized-routine-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_routine_project" + id = "auth_routine_dataset" +} + +resource "google_bigquery_routine" "public-routine" { + dataset_id = module.bigquery-authorized-authorized-routine-dataset-public.dataset_id + routine_id = "auth_routine" + routine_type = "TABLE_VALUED_FUNCTION" + language = "SQL" + definition_body = <<-EOS + SELECT 1 + value AS value + EOS + arguments { + name = "value" + argument_kind = "FIXED_TYPE" + data_type = jsonencode({ "typeKind" = "INT64" }) + } + return_table_type = jsonencode({ "columns" = [ + { "name" = "value", "type" = { "typeKind" = "INT64" } }, + ] }) +} +# tftest modules=4 resources=9 inventory=authorized_resources.yaml +``` + +Authorized views can be specified both using the standard `access` options and the `authorized_views` blocks. The example configuration below uses both blocks, and will create a dataset with three authorized views `view_id_1`, `view_id_2`, and `view_id_3`. + +```hcl +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my-dataset" + authorized_views = [ + { + project_id = "view_project" + dataset_id = "view_dataset" + table_id = "view_id_1" + }, + { + project_id = "view_project" + dataset_id = "view_dataset" + table_id = "view_id_2" + } + ] + access = { + view_2 = { role = "READER", type = "view" } + view_3 = { role = "READER", type = "view" } + } + access_identities = { + view_2 = "view_project|view_dataset|view_id_2" + view_3 = "view_project|view_dataset|view_id_3" + } +} +# tftest modules=1 resources=4 inventory=authorized_resources_views.yaml +``` + ### Dataset options Dataset options are set via the `options` variable. all options must be specified, but a `null` value can be set to options that need to use defaults. @@ -178,20 +291,23 @@ module "bigquery-dataset" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [id](variables.tf#L69) | Dataset id. | string | ✓ | | -| [project_id](variables.tf#L99) | Id of the project where datasets will be created. | string | ✓ | | +| [id](variables.tf#L98) | Dataset id. | string | ✓ | | +| [project_id](variables.tf#L128) | Id of the project where datasets will be created. | string | ✓ | | | [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…})) | | {} | | [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | -| [dataset_access](variables.tf#L39) | Set access in the dataset resource instead of using separate resources. | bool | | false | -| [description](variables.tf#L45) | Optional description. | string | | "Terraform managed." | -| [encryption_key](variables.tf#L51) | Self link of the KMS key that will be used to protect destination table. | string | | null | -| [friendly_name](variables.tf#L57) | Dataset friendly name. | string | | null | -| [iam](variables.tf#L63) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | -| [labels](variables.tf#L74) | Dataset labels. | map(string) | | {} | -| [location](variables.tf#L80) | Dataset location. | string | | "EU" | -| [options](variables.tf#L86) | Dataset options. | object({…}) | | {} | -| [tables](variables.tf#L104) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | -| [views](variables.tf#L132) | View definitions. | map(object({…})) | | {} | +| [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset. | list(object({…})) | | [] | +| [authorized_routines](variables.tf#L48) | An array of authorized routine to be authorized on the dataset. | list(object({…})) | | [] | +| [authorized_views](variables.tf#L58) | An array of views to be authorized on the dataset. | list(object({…})) | | [] | +| [dataset_access](variables.tf#L68) | Set access in the dataset resource instead of using separate resources. | bool | | false | +| [description](variables.tf#L74) | Optional description. | string | | "Terraform managed." | +| [encryption_key](variables.tf#L80) | Self link of the KMS key that will be used to protect destination table. | string | | null | +| [friendly_name](variables.tf#L86) | Dataset friendly name. | string | | null | +| [iam](variables.tf#L92) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | +| [labels](variables.tf#L103) | Dataset labels. | map(string) | | {} | +| [location](variables.tf#L109) | Dataset location. | string | | "EU" | +| [options](variables.tf#L115) | Dataset options. | object({…}) | | {} | +| [tables](variables.tf#L133) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | +| [views](variables.tf#L161) | View definitions. | map(object({…})) | | {} | ## Outputs @@ -199,11 +315,11 @@ module "bigquery-dataset" { |---|---|:---:| | [dataset](outputs.tf#L17) | Dataset resource. | | | [dataset_id](outputs.tf#L22) | Dataset id. | | -| [id](outputs.tf#L34) | Fully qualified dataset id. | | -| [self_link](outputs.tf#L46) | Dataset self link. | | -| [table_ids](outputs.tf#L58) | Map of fully qualified table ids keyed by table ids. | | -| [tables](outputs.tf#L63) | Table resources. | | -| [view_ids](outputs.tf#L68) | Map of fully qualified view ids keyed by view ids. | | -| [views](outputs.tf#L73) | View resources. | | +| [id](outputs.tf#L36) | Fully qualified dataset id. | | +| [self_link](outputs.tf#L50) | Dataset self link. | | +| [table_ids](outputs.tf#L64) | Map of fully qualified table ids keyed by table ids. | | +| [tables](outputs.tf#L69) | Table resources. | | +| [view_ids](outputs.tf#L74) | Map of fully qualified view ids keyed by view ids. | | +| [views](outputs.tf#L79) | View resources. | | diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index 62d75224..0a66b829 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -20,15 +20,23 @@ locals { access_special = { for k, v in var.access : k => v if v.type == "special_group" } access_user = { for k, v in var.access : k => v if v.type == "user" } access_view = { for k, v in var.access : k => v if v.type == "view" } + identities_view = { for k, v in local.access_view : k => try( zipmap( - ["project", "dataset", "table"], + ["project_id", "dataset_id", "table_id"], split("|", var.access_identities[k]) ), - { project = null, dataset = null, table = null } + { project_id = null, dataset_id = null, table_id = null } ) } + + authorized_views = merge( + { for access_key, view in local.identities_view : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }, + { for view in var.authorized_views : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }) + authorized_datasets = { for dataset in var.authorized_datasets : "${dataset["project_id"]}_${dataset["dataset_id"]}" => dataset } + authorized_routines = { for routine in var.authorized_routines : "${routine["project_id"]}_${routine["dataset_id"]}_${routine["routine_id"]}" => routine } + } resource "google_bigquery_dataset" "default" { @@ -78,12 +86,36 @@ resource "google_bigquery_dataset" "default" { } dynamic "access" { - for_each = var.dataset_access ? local.access_view : {} + for_each = var.dataset_access ? local.authorized_views : {} content { view { - project_id = local.identities_view[access.key].project - dataset_id = local.identities_view[access.key].dataset - table_id = local.identities_view[access.key].table + project_id = each.value.project_id + dataset_id = each.value.dataset_id + table_id = each.value.table_id + } + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.authorized_datasets : {} + content { + dataset { + dataset { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + } + target_types = ["VIEWS"] + } + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.authorized_routines : {} + content { + routine { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + routine_id = each.value.routine_id } } } @@ -132,15 +164,38 @@ resource "google_bigquery_dataset_access" "user_by_email" { user_by_email = try(var.access_identities[each.key]) } -resource "google_bigquery_dataset_access" "views" { - for_each = var.dataset_access ? {} : local.access_view - provider = google-beta +resource "google_bigquery_dataset_access" "authorized_views" { + for_each = var.dataset_access ? {} : local.authorized_views project = var.project_id dataset_id = google_bigquery_dataset.default.dataset_id view { - project_id = local.identities_view[each.key].project - dataset_id = local.identities_view[each.key].dataset - table_id = local.identities_view[each.key].table + project_id = each.value.project_id + dataset_id = each.value.dataset_id + table_id = each.value.table_id + } +} + +resource "google_bigquery_dataset_access" "authorized_datasets" { + for_each = var.dataset_access ? {} : local.authorized_datasets + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + dataset { + dataset { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + } + target_types = ["VIEWS"] + } +} + +resource "google_bigquery_dataset_access" "authorized_routines" { + for_each = var.dataset_access ? {} : local.authorized_routines + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + routine { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + routine_id = each.value.routine_id } } diff --git a/modules/bigquery-dataset/outputs.tf b/modules/bigquery-dataset/outputs.tf index dd2da22c..5c2ee466 100644 --- a/modules/bigquery-dataset/outputs.tf +++ b/modules/bigquery-dataset/outputs.tf @@ -23,11 +23,13 @@ output "dataset_id" { description = "Dataset id." value = google_bigquery_dataset.default.dataset_id depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, google_bigquery_dataset_access.domain, google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, - google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.views + google_bigquery_dataset_access.user_by_email ] } @@ -35,11 +37,13 @@ output "id" { description = "Fully qualified dataset id." value = google_bigquery_dataset.default.id depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, google_bigquery_dataset_access.domain, google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, - google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.views + google_bigquery_dataset_access.user_by_email ] } @@ -47,11 +51,13 @@ output "self_link" { description = "Dataset self link." value = google_bigquery_dataset.default.self_link depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, google_bigquery_dataset_access.domain, google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, - google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.views + google_bigquery_dataset_access.user_by_email ] } diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index c738845e..66eb8934 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -36,6 +36,35 @@ variable "access_identities" { default = {} } +variable "authorized_datasets" { + description = "An array of datasets to be authorized on the dataset." + type = list(object({ + dataset_id = string, + project_id = string, + })) + default = [] +} + +variable "authorized_routines" { + description = "An array of authorized routine to be authorized on the dataset." + type = list(object({ + project_id = string, + dataset_id = string, + routine_id = string + })) + default = [] +} + +variable "authorized_views" { + description = "An array of views to be authorized on the dataset." + type = list(object({ + dataset_id = string, + project_id = string, + table_id = string # this is the view id, but we keep table_id to stay consistent as the resource + })) + default = [] +} + variable "dataset_access" { description = "Set access in the dataset resource instead of using separate resources." type = bool diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml new file mode 100644 index 00000000..518ce6fc --- /dev/null +++ b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml @@ -0,0 +1,42 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.bigquery-dataset-private.google_bigquery_dataset.default: + dataset_id: private_dataset + project: private_project + module.bigquery-dataset-private.google_bigquery_dataset_access.authorized_datasets["auth_dataset_project_auth_dataset"]: + dataset: + - dataset: + - dataset_id: auth_dataset + project_id: auth_dataset_project + target_types: ["VIEWS"] + module.bigquery-dataset-private.google_bigquery_dataset_access.authorized_routines["auth_routine_project_auth_routine_dataset_auth_routine"]: + routine: + - dataset_id: auth_routine_dataset + project_id: auth_routine_project + routine_id: auth_routine + module.bigquery-dataset-private.google_bigquery_dataset_access.authorized_views["auth_view_project_auth_view_dataset_auth_view"]: + view: + - dataset_id: auth_view_dataset + project_id: auth_view_project + table_id: auth_view + +counts: + google_bigquery_dataset: 4 + google_bigquery_dataset_access: 3 + google_bigquery_routine: 1 + google_bigquery_table: 1 + modules: 4 + resources: 9 diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml new file mode 100644 index 00000000..25f0b3a8 --- /dev/null +++ b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.bigquery-dataset.google_bigquery_dataset.default: + dataset_id: my-dataset + project: my-project + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_1"]: + view: + - dataset_id: view_dataset + project_id: view_project + table_id: view_id_1 + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_2"]: + view: + - dataset_id: view_dataset + project_id: view_project + table_id: view_id_2 + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_3"]: + view: + - dataset_id: view_dataset + project_id: view_project + table_id: view_id_3 +counts: + google_bigquery_dataset: 1 + google_bigquery_dataset_access: 3 diff --git a/tests/modules/bigquery_dataset/examples/simple.yaml b/tests/modules/bigquery_dataset/examples/simple.yaml index acf8e819..490778ad 100644 --- a/tests/modules/bigquery_dataset/examples/simple.yaml +++ b/tests/modules/bigquery_dataset/examples/simple.yaml @@ -33,7 +33,7 @@ values: project: my-project role: OWNER user_by_email: ludo@ludomagno.net - module.bigquery-dataset.google_bigquery_dataset_access.views["view_1"]: + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["my-project_my-dataset_my-table"]: dataset_id: my-dataset project: my-project view: