zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor access in bq module, add iam (#172)

This commit is contained in:
Ludovico Magnocavallo 2020-11-21 09:45:56 +01:00 committed by GitHub
parent 2ea59d2ee4
commit cffc823f48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 89 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cloud-operations/scheduled-asset-inventory-export-bq/main.tf
View File

@ -125,8 +125,8 @@ module "bq" {
source = "../../modules/bigquery-dataset"
project_id = module.project.project_id
id = var.cai_config.bq_dataset
access_roles = {
owner = { role = "OWNER", type = "user_by_email" }
access = {
owner = { role = "OWNER", type = "user" }
}
access_identities = {
owner = module.service-account.email

6
data-solutions/gcs-to-bq-with-dataflow/main.tf
View File

@ -283,9 +283,9 @@ module "bigquery-dataset" {
source = "../../modules/bigquery-dataset"
project_id = module.project-service.project_id
id = "bq_dataset"
access_roles = {
reader-group = { role = "READER", type = "service_account" }
owner = { role = "OWNER", type = "user_by_email" }
access = {
reader-group = { role = "READER", type = "user" }
owner = { role = "OWNER", type = "user" }
}
access_identities = {
reader-group = module.service-account-bq.email

40
modules/bigquery-dataset/README.md
View File

@ -22,18 +22,40 @@ module "bigquery-dataset" {
source = "./modules/bigquery-dataset"
project_id = "my-project"
id = "my-dataset"
access_roles = {
reader-group = { role = "READER", type = "group_by_email" }
owner = { role = "OWNER", type = "user_by_email" }
access = {
reader-group = { role = "READER", type = "group" }
owner = { role = "OWNER", type = "user" }
project_owners = { role = "OWNER", type = "special_group" }
view_1 = { role = "READER", type = "view" }
}
access_identities = {
reader-group = "playground-test@ludomagno.net"
owner = "ludo@ludomagno.net"
reader-group = "playground-test@ludomagno.net"
owner = "ludo@ludomagno.net"
project_owners = "projectOwners"
view_1 = "my-project|my-dataset|my-table"
}
}
# tftest:modules=1:resources=3
# tftest:modules=1:resources=5
```
### IAM roles
Access configuration can also be specified via IAM instead of basic roles via the `iam` variable. When using IAM, basic roles cannot be used via the `access` family variables.
```hcl
module "bigquery-dataset" {
source = "./modules/bigquery-dataset"
project_id = "my-project"
id = "my-dataset"
iam = {
"roles/bigquery.dataOwner" = ["user:user1@example.org"]
}
}
# tftest:modules=1:resources=2
```
roles/bigquery.dataOwner
### Dataset options
Dataset options are set via the `options` variable. all options must be specified, but a `null` value can be set to options that need to use defaults.
@ -137,12 +159,12 @@ module "bigquery-dataset" {
|---|---|:---: |:---:|:---:|
| id | Dataset id. | <code title="">string</code> | ✓ | |
| project_id | Id of the project where datasets will be created. | <code title="">string</code> | ✓ | |
| *access_identities* | Map of access identities used for access roles with type different from `view`. A separate variable is needed as identities can be set to dynamic values. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *access_roles* | Map of access rules with role and identity type. Keys are arbitrary and only used to combine identities with each role. Valid types are `domain`, `group_by_email`, `special_group`, `user_by_email`, `view`. | <code title="map&#40;object&#40;&#123;&#10;role &#61; string&#10;type &#61; string&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *access_views* | Map of view data for access roles with identity type equal to `view`. A separate variable is needed as identities can be set to dynamic values. | <code title="map&#40;object&#40;&#123;&#10;project_id &#61; string&#10;dataset_id &#61; string&#10;table_id &#61; string&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="">{}</code> |
| *access* | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | <code title="map&#40;object&#40;&#123;&#10;role &#61; string&#10;type &#61; string&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="&#123;&#125;&#10;validation &#123;&#10;condition &#61; can&#40;&#91;&#10;for k, v in var.access :&#10;index&#40;&#91;&#34;OWNER&#34;, &#34;READER&#34;, &#34;WRITER&#34;&#93;, v.role&#41;&#10;&#93;&#41;&#10;error_message &#61; &#34;Access role must be one of &#39;OWNER&#39;, &#39;READER&#39;, &#39;WRITER&#39;.&#34;&#10;&#125;&#10;validation &#123;&#10;condition &#61; can&#40;&#91;&#10;for k, v in var.access :&#10;index&#40;&#91;&#34;domain&#34;, &#34;group&#34;, &#34;special_group&#34;, &#34;user&#34;, &#34;view&#34;&#93;, v.type&#41;&#10;&#93;&#41;&#10;error_message &#61; &#34;Access type must be one of &#39;domain&#39;, &#39;group&#39;, &#39;special_group&#39;, &#39;user&#39;, &#39;view&#39;.&#34;&#10;&#125;">...</code> |
| *access_identities* | Map of access identities used for basic access roles. View identities have the format 'project_id|dataset_id|table_id'. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *dataset_access* | Set access in the dataset resource instead of using separate resources. | <code title="">bool</code> | | <code title="">false</code> |
| *encryption_key* | Self link of the KMS key that will be used to protect destination table. | <code title="">string</code> | | <code title="">null</code> |
| *friendly_name* | Dataset friendly name. | <code title="">string</code> | | <code title="">null</code> |
| *iam* | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | <code title="map&#40;list&#40;string&#41;&#41;">map(list(string))</code> | | <code title="">{}</code> |
| *labels* | Dataset labels. | <code title="map&#40;string&#41;">map(string)</code> | | <code title="">{}</code> |
| *location* | Dataset location. | <code title="">string</code> | | <code title="">EU</code> |
| *options* | Dataset options. | <code title="object&#40;&#123;&#10;default_table_expiration_ms &#61; number&#10;default_partition_expiration_ms &#61; number&#10;delete_contents_on_destroy &#61; bool&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;default_table_expiration_ms &#61; null&#10;default_partition_expiration_ms &#61; null&#10;delete_contents_on_destroy &#61; false&#10;&#125;">...</code> |

48
modules/bigquery-dataset/main.tf
View File

@ -15,20 +15,19 @@
*/
locals {
access_domain = {
for k, v in var.access_roles : k => v if v.type == "domain"
}
access_group = {
for k, v in var.access_roles : k => v if v.type == "group_by_email"
}
access_special = {
for k, v in var.access_roles : k => v if v.type == "special_group"
}
access_user = {
for k, v in var.access_roles : k => v if v.type == "user_by_email"
}
access_view = {
for k, v in var.access_roles : k => v if v.type == "view"
access_domain = { for k, v in var.access : k => v if v.type == "domain" }
access_group = { for k, v in var.access : k => v if v.type == "group" }
access_special = { for k, v in var.access : k => v if v.type == "special_group" }
access_user = { for k, v in var.access : k => v if v.type == "user" }
access_view = { for k, v in var.access : k => v if v.type == "view" }
identities_view = {
for k, v in local.access_view : k => try(
zipmap(
["project", "dataset", "table"],
split("|", var.access_identities[k])
),
{ project = null, dataset = null, table = null }
)
}
}
@ -80,9 +79,9 @@ resource "google_bigquery_dataset" "default" {
for_each = var.dataset_access ? local.access_view : {}
content {
view {
project_id = try(var.access.views[access.key].project_id, null)
dataset_id = try(var.access.views[access.key].dataset_id, null)
table_id = try(var.access.views[access.key].table_id, null)
project_id = local.identities_view[access.key].project
dataset_id = local.identities_view[access.key].dataset
table_id = local.identities_view[access.key].table
}
}
}
@ -95,7 +94,6 @@ resource "google_bigquery_dataset" "default" {
}
}
resource "google_bigquery_dataset_access" "domain" {
for_each = var.dataset_access ? {} : local.access_domain
provider = google-beta
@ -138,12 +136,19 @@ resource "google_bigquery_dataset_access" "views" {
project = var.project_id
dataset_id = google_bigquery_dataset.default.dataset_id
view {
project_id = try(var.access_views[each.key].project_id, null)
dataset_id = try(var.access_views[each.key].dataset_id, null)
table_id = try(var.access_views[each.key].table_id, null)
project_id = local.identities_view[each.key].project
dataset_id = local.identities_view[each.key].dataset
table_id = local.identities_view[each.key].table
}
}
resource "google_bigquery_dataset_iam_binding" "bindings" {
for_each = var.iam
dataset_id = google_bigquery_dataset.default.dataset_id
role = each.key
members = each.value
}
resource "google_bigquery_table" "default" {
provider = google-beta
for_each = var.tables
@ -187,7 +192,6 @@ resource "google_bigquery_table" "default" {
}
resource "google_bigquery_table" "views" {
depends_on = [google_bigquery_table.default]
for_each = var.views

42
modules/bigquery-dataset/variables.tf
View File

@ -14,29 +14,33 @@
* limitations under the License.
*/
variable "access_identities" {
description = "Map of access identities used for access roles with type different from `view`. A separate variable is needed as identities can be set to dynamic values."
type = map(string)
default = {}
}
variable "access_roles" {
description = "Map of access rules with role and identity type. Keys are arbitrary and only used to combine identities with each role. Valid types are `domain`, `group_by_email`, `special_group`, `user_by_email`, `view`."
variable "access" {
description = "Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`."
type = map(object({
role = string
type = string
}))
default = {}
validation {
condition = can([
for k, v in var.access :
index(["OWNER", "READER", "WRITER"], v.role)
])
error_message = "Access role must be one of 'OWNER', 'READER', 'WRITER'."
}
validation {
condition = can([
for k, v in var.access :
index(["domain", "group", "special_group", "user", "view"], v.type)
])
error_message = "Access type must be one of 'domain', 'group', 'special_group', 'user', 'view'."
}
}
variable "access_views" {
description = "Map of view data for access roles with identity type equal to `view`. A separate variable is needed as identities can be set to dynamic values."
type = map(object({
project_id = string
dataset_id = string
table_id = string
}))
default = {}
variable "access_identities" {
description = "Map of access identities used for basic access roles. View identities have the format 'project_id|dataset_id|table_id'."
type = map(string)
default = {}
}
variable "dataset_access" {
@ -51,6 +55,12 @@ variable "encryption_key" {
default = null
}
variable "iam" {
description = "IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles."
type = map(list(string))
default = {}
}
variable "labels" {
description = "Dataset labels."
type = map(string)

2
tests/data_solutions/gcs_to_bq_with_dataflow/test_plan.py
View File

@ -24,4 +24,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner(FIXTURES_DIR)
assert len(modules) == 14
assert len(resources) == 60
assert len(resources) == 61