zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #271 from terraform-google-modules/df-cmek-01 

Add support for CMEK keys in Data Foundation end to end example
This commit is contained in:
lcaggio 2021-07-08 08:22:19 +02:00 committed by GitHub
parent cdedc8dd4e d760959999
commit 37b19ec330
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 65 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
## [Unreleased]
- create `pubsub` service identity if service is enabled
- support for creation of GKE Autopilot clusters
- Add support for CMEK keys in Data Foundation end to end example
## [5.0.0] - 2021-06-17

2
data-solutions/data-platform-foundations/01-environment/README.md
View File

@ -43,6 +43,7 @@ Once done testing, you can clean up resources by running `terraform destroy`.
| *prefix* | Prefix used to generate project id and name. | <code title="">string</code> | | <code title="">null</code> |
| *project_names* | Override this variable if you need non-standard names. | <code title="object&#40;&#123;&#10;datamart &#61; string&#10;dwh &#61; string&#10;landing &#61; string&#10;services &#61; string&#10;transformation &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;datamart &#61; &#34;datamart&#34;&#10;dwh &#61; &#34;datawh&#34;&#10;landing &#61; &#34;landing&#34;&#10;services &#61; &#34;services&#34;&#10;transformation &#61; &#34;transformation&#34;&#10;&#125;">...</code> |
| *service_account_names* | Override this variable if you need non-standard names. | <code title="object&#40;&#123;&#10;main &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;main &#61; &#34;data-platform-main&#34;&#10;&#125;">...</code> |
| *service_encryption_key_ids* | Cloud KMS encryption key in {LOCATION => [KEY_URL]} format. Keys belong to existing project. | <code title="object&#40;&#123;&#10;multiregional &#61; string&#10;global &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;multiregional &#61; null&#10;global &#61; null&#10;&#125;">...</code> |
## Outputs
@ -50,4 +51,5 @@ Once done testing, you can clean up resources by running `terraform destroy`.
|---|---|:---:|
| project_ids | Project ids for created projects. | |
| service_account | Main service account. | |
| service_encryption_key_ids | Cloud KMS encryption keys in {LOCATION => [KEY_URL]} format. | |
<!-- END TFDOC -->

28
data-solutions/data-platform-foundations/01-environment/main.tf
View File

@ -25,16 +25,19 @@ module "project-datamart" {
prefix = var.prefix
name = var.project_names.datamart
services = [
"bigtable.googleapis.com",
"bigtableadmin.googleapis.com",
"bigquery.googleapis.com",
"bigquerystorage.googleapis.com",
"bigqueryreservation.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com",
]
iam = {
"roles/editor" = [module.sa-services-main.iam_email]
}
service_encryption_key_ids = {
bq = [var.service_encryption_key_ids.multiregional]
storage = [var.service_encryption_key_ids.multiregional]
}
}
module "project-dwh" {
@ -47,11 +50,16 @@ module "project-dwh" {
"bigquery.googleapis.com",
"bigquerystorage.googleapis.com",
"bigqueryreservation.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com",
]
iam = {
"roles/editor" = [module.sa-services-main.iam_email]
}
service_encryption_key_ids = {
bq = [var.service_encryption_key_ids.multiregional]
storage = [var.service_encryption_key_ids.multiregional]
}
}
module "project-landing" {
@ -62,11 +70,16 @@ module "project-landing" {
name = var.project_names.landing
services = [
"pubsub.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com",
]
iam = {
"roles/editor" = [module.sa-services-main.iam_email]
}
service_encryption_key_ids = {
pubsub = [var.service_encryption_key_ids.global]
storage = [var.service_encryption_key_ids.multiregional]
}
}
module "project-services" {
@ -76,14 +89,19 @@ module "project-services" {
prefix = var.prefix
name = var.project_names.services
services = [
"storage.googleapis.com",
"storage-component.googleapis.com",
"sourcerepo.googleapis.com",
"stackdriver.googleapis.com",
"cloudasset.googleapis.com",
"cloudkms.googleapis.com"
]
iam = {
"roles/editor" = [module.sa-services-main.iam_email]
}
service_encryption_key_ids = {
storage = [var.service_encryption_key_ids.multiregional]
}
}
module "project-transformation" {
@ -97,11 +115,17 @@ module "project-transformation" {
"compute.googleapis.com",
"dataflow.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
"storage-component.googleapis.com",
]
iam = {
"roles/editor" = [module.sa-services-main.iam_email]
}
service_encryption_key_ids = {
compute = [var.service_encryption_key_ids.global]
storage = [var.service_encryption_key_ids.multiregional]
dataflow = [var.service_encryption_key_ids.global]
}
}
###############################################################################

5
data-solutions/data-platform-foundations/01-environment/outputs.tf
View File

@ -25,6 +25,11 @@ output "project_ids" {
}
}
output "service_encryption_key_ids" {
description = "Cloud KMS encryption keys in {LOCATION => [KEY_URL]} format."
value = var.service_encryption_key_ids
}
output "service_account" {
description = "Main service account."
value = module.sa-services-main.email

12
data-solutions/data-platform-foundations/01-environment/variables.tf
View File

@ -55,3 +55,15 @@ variable "service_account_names" {
main = "data-platform-main"
}
}
variable "service_encryption_key_ids" {
description = "Cloud KMS encryption key in {LOCATION => [KEY_URL]} format. Keys belong to existing project."
type = object({
multiregional = string
global = string
})
default = {
multiregional = null
global = null
}
}

1
data-solutions/data-platform-foundations/02-resources/README.md
View File

@ -61,6 +61,7 @@ Once done testing, you can clean up resources by running `terraform destroy`.
| *landing_pubsub* | List of landing pubsub topics and subscriptions to create | <code title="map&#40;map&#40;object&#40;&#123;&#10;iam &#61; map&#40;list&#40;string&#41;&#41;&#10;labels &#61; map&#40;string&#41;&#10;options &#61; object&#40;&#123;&#10;ack_deadline_seconds &#61; number&#10;message_retention_duration &#61; number&#10;retain_acked_messages &#61; bool&#10;expiration_policy_ttl &#61; number&#10;&#125;&#41;&#10;&#125;&#41;&#41;&#41;">map(map(object({...})))</code> | | <code title="&#123;&#10;landing-1 &#61; &#123;&#10;sub1 &#61; &#123;&#10;iam &#61; &#123;&#10;&#125;&#10;labels &#61; &#123;&#125;&#10;options &#61; null&#10;&#125;&#10;sub2 &#61; &#123;&#10;iam &#61; &#123;&#125;&#10;labels &#61; &#123;&#125;,&#10;options &#61; null&#10;&#125;,&#10;&#125;&#10;&#125;">...</code> |
| *landing_service_account* | landing service accounts list. | <code title="">string</code> | | <code title="">sa-landing</code> |
| *service_account_names* | Project service accounts list. | <code title="object&#40;&#123;&#10;datamart &#61; string&#10;dwh &#61; string&#10;landing &#61; string&#10;services &#61; string&#10;transformation &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;datamart &#61; &#34;sa-datamart&#34;&#10;dwh &#61; &#34;sa-datawh&#34;&#10;landing &#61; &#34;sa-landing&#34;&#10;services &#61; &#34;sa-services&#34;&#10;transformation &#61; &#34;sa-transformation&#34;&#10;&#125;">...</code> |
| *service_encryption_key_ids* | Cloud KMS encryption key in {LOCATION => [KEY_URL]} format. Keys belong to existing project. | <code title="object&#40;&#123;&#10;multiregional &#61; string&#10;global &#61; string&#10;&#125;&#41;">object({...})</code> | | <code title="&#123;&#10;multiregional &#61; null&#10;global &#61; null&#10;&#125;">...</code> |
| *transformation_buckets* | List of transformation buckets to create | <code title="map&#40;object&#40;&#123;&#10;location &#61; string&#10;name &#61; string&#10;&#125;&#41;&#41;">map(object({...}))</code> | | <code title="&#123;&#10;temp &#61; &#123;&#10;location &#61; &#34;EU&#34;&#10;name &#61; &#34;temp&#34;&#10;&#125;,&#10;templates &#61; &#123;&#10;location &#61; &#34;EU&#34;&#10;name &#61; &#34;templates&#34;&#10;&#125;,&#10;&#125;">...</code> |
| *transformation_subnets* | List of subnets to create in the transformation Project. | <code title="list&#40;object&#40;&#123;&#10;ip_cidr_range &#61; string&#10;name &#61; string&#10;region &#61; string&#10;secondary_ip_range &#61; map&#40;string&#41;&#10;&#125;&#41;&#41;">list(object({...}))</code> | | <code title="&#91;&#10;&#123;&#10;ip_cidr_range &#61; &#34;10.1.0.0&#47;20&#34;&#10;name &#61; &#34;transformation-subnet&#34;&#10;region &#61; &#34;europe-west3&#34;&#10;secondary_ip_range &#61; &#123;&#125;&#10;&#125;,&#10;&#93;">...</code> |
| *transformation_vpc_name* | Name of the VPC created in the transformation Project. | <code title="">string</code> | | <code title="">transformation-vpc</code> |

5
data-solutions/data-platform-foundations/02-resources/main.tf
View File

@ -85,6 +85,7 @@ module "landing-buckets" {
"roles/storage.objectCreator" = [module.landing-sa.iam_email]
"roles/storage.admin" = [module.transformation-sa.iam_email]
}
encryption_key = var.service_encryption_key_ids.multiregional
}
module "transformation-buckets" {
@ -97,6 +98,7 @@ module "transformation-buckets" {
iam = {
"roles/storage.admin" = [module.transformation-sa.iam_email]
}
encryption_key = var.service_encryption_key_ids.multiregional
}
###############################################################################
@ -116,6 +118,7 @@ module "datamart-bq" {
: v
)
}
encryption_key = var.service_encryption_key_ids.multiregional
}
module "dwh-bq" {
@ -131,6 +134,7 @@ module "dwh-bq" {
: v
)
}
encryption_key = var.service_encryption_key_ids.multiregional
}
###############################################################################
@ -160,4 +164,5 @@ module "landing-pubsub" {
"roles/pubsub.subscriber" = [module.transformation-sa.iam_email]
})
}
kms_key = var.service_encryption_key_ids.global
}

12
data-solutions/data-platform-foundations/02-resources/variables.tf
View File

@ -169,3 +169,15 @@ variable "transformation_vpc_name" {
type = string
default = "transformation-vpc"
}
variable "service_encryption_key_ids" {
description = "Cloud KMS encryption key in {LOCATION => [KEY_URL]} format. Keys belong to existing project."
type = object({
multiregional = string
global = string
})
default = {
multiregional = null
global = null
}
}

2
tests/data_solutions/data_platform_foundations/test_plan.py
View File

@ -24,4 +24,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner(FIXTURES_DIR)
assert len(modules) == 6
assert len(resources) == 35
assert len(resources) == 44