zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into binauthz

This commit is contained in:
Ludovico Magnocavallo 2022-06-18 08:54:16 +02:00 committed by GitHub
parent 2e9fdea1a4 b04b63ee79
commit 3227bd13ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 277 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -6,12 +6,16 @@ All notable changes to this project will be documented in this file.
- add support for IAM and Cloud Build triggers to source repository module - add support for IAM and Cloud Build triggers to source repository module
- add `id` output to service account module - add `id` output to service account module
- add support for secrets to cloud function module
- new binary authorization module
**FAST** **FAST**
- add support for Cloud Source Repositories in stage 0 and 1 CI/CD - add support for Cloud Source Repositories in stage 0 and 1 CI/CD
- fix Gitlab workflow indentation - fix Gitlab workflow indentation
- remove unsupported attributes and add supported ones to the Gitlab mapping used for Workload Identity Federation pools - remove unsupported attributes and add supported ones to the Gitlab mapping used for Workload Identity Federation pools
- add roles for CI/CD source repositories to stage 1 service account on automation project
- fixes to CI/CD source repositories in stage 1
## [16.0.0] - 2022-06-06 ## [16.0.0] - 2022-06-06

2
examples/factories/project-factory/main.tf
View File

@ -28,7 +28,7 @@ locals {
_service_accounts_iam = { _service_accounts_iam = {
for r in local._service_accounts_iam_bindings : r => [ for r in local._service_accounts_iam_bindings : r => [
for k, v in var.service_accounts : for k, v in var.service_accounts :
"serviceAccount:${k}@${local._project_id}.iam.gserviceaccount.com" module.service-accounts[k].iam_email
if try(index(v, r), null) != null if try(index(v, r), null) != null
] ]
} }

20
fast/stages/00-bootstrap/README.md
View File

@ -457,15 +457,15 @@ The remaining configuration is manual, as it regards the repositories themselves
| name | description | sensitive | consumers | | name | description | sensitive | consumers |
|---|---|:---:|---| |---|---|:---:|---|
| [automation](outputs.tf#L81) | Automation resources. | | | | [automation](outputs.tf#L82) | Automation resources. | | |
| [billing_dataset](outputs.tf#L86) | BigQuery dataset prepared for billing export. | | | | [billing_dataset](outputs.tf#L87) | BigQuery dataset prepared for billing export. | | |
| [cicd_repositories](outputs.tf#L91) | CI/CD repository configurations. | | | | [cicd_repositories](outputs.tf#L92) | CI/CD repository configurations. | | |
| [custom_roles](outputs.tf#L103) | Organization-level custom roles. | | | | [custom_roles](outputs.tf#L104) | Organization-level custom roles. | | |
| [federated_identity](outputs.tf#L108) | Workload Identity Federation pool and providers. | | | | [federated_identity](outputs.tf#L109) | Workload Identity Federation pool and providers. | | |
| [outputs_bucket](outputs.tf#L118) | GCS bucket where generated output files are stored. | | | | [outputs_bucket](outputs.tf#L119) | GCS bucket where generated output files are stored. | | |
| [project_ids](outputs.tf#L123) | Projects created by this stage. | | | | [project_ids](outputs.tf#L124) | Projects created by this stage. | | |
| [providers](outputs.tf#L142) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> | | [providers](outputs.tf#L143) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
| [service_accounts](outputs.tf#L132) | Automation service accounts created by this stage. | | | | [service_accounts](outputs.tf#L133) | Automation service accounts created by this stage. | | |
| [tfvars](outputs.tf#L151) | Terraform variable files for the following stages. | ✓ | | | [tfvars](outputs.tf#L152) | Terraform variable files for the following stages. | ✓ | |
<!-- END TFDOC --> <!-- END TFDOC -->

6
fast/stages/00-bootstrap/automation.tf
View File

@ -38,12 +38,18 @@ module "automation-project" {
"roles/owner" = [ "roles/owner" = [
module.automation-tf-bootstrap-sa.iam_email module.automation-tf-bootstrap-sa.iam_email
] ]
"roles/cloudbuild.builds.editor" = [
module.automation-tf-resman-sa.iam_email
]
"roles/iam.serviceAccountAdmin" = [ "roles/iam.serviceAccountAdmin" = [
module.automation-tf-resman-sa.iam_email module.automation-tf-resman-sa.iam_email
] ]
"roles/iam.workloadIdentityPoolAdmin" = [ "roles/iam.workloadIdentityPoolAdmin" = [
module.automation-tf-resman-sa.iam_email module.automation-tf-resman-sa.iam_email
] ]
"roles/source.admin" = [
module.automation-tf-resman-sa.iam_email
]
"roles/storage.admin" = [ "roles/storage.admin" = [
module.automation-tf-resman-sa.iam_email module.automation-tf-resman-sa.iam_email
] ]

1
fast/stages/00-bootstrap/outputs.tf
View File

@ -57,6 +57,7 @@ locals {
federated_identity_providers = local.wif_providers federated_identity_providers = local.wif_providers
outputs_bucket = module.automation-tf-output-gcs.name outputs_bucket = module.automation-tf-output-gcs.name
project_id = module.automation-project.project_id project_id = module.automation-project.project_id
project_number = module.automation-project.number
} }
custom_roles = local.custom_roles custom_roles = local.custom_roles
} }

40
fast/stages/01-resman/README.md
View File

@ -178,30 +178,30 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
| name | description | type | required | default | producer | | name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:| |---|---|:---:|:---:|:---:|:---:|
| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10; outputs_bucket &#61; string&#10; project_id &#61; string&#10; federated_identity_pool &#61; string&#10; federated_identity_providers &#61; map&#40;object&#40;&#123;&#10; issuer &#61; string&#10; issuer_uri &#61; string&#10; name &#61; string&#10; principal_tpl &#61; string&#10; principalset_tpl &#61; string&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>00-bootstrap</code> | | [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10; outputs_bucket &#61; string&#10; project_id &#61; string&#10; project_number &#61; string&#10; federated_identity_pool &#61; string&#10; federated_identity_providers &#61; map&#40;object&#40;&#123;&#10; issuer &#61; string&#10; issuer_uri &#61; string&#10; name &#61; string&#10; principal_tpl &#61; string&#10; principalset_tpl &#61; string&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>00-bootstrap</code> |
| [billing_account](variables.tf#L37) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object&#40;&#123;&#10; id &#61; string&#10; organization_id &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>00-bootstrap</code> | | [billing_account](variables.tf#L38) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object&#40;&#123;&#10; id &#61; string&#10; organization_id &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>00-bootstrap</code> |
| [organization](variables.tf#L140) | Organization details. | <code title="object&#40;&#123;&#10; domain &#61; string&#10; id &#61; number&#10; customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>00-bootstrap</code> | | [organization](variables.tf#L141) | Organization details. | <code title="object&#40;&#123;&#10; domain &#61; string&#10; id &#61; number&#10; customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>00-bootstrap</code> |
| [prefix](variables.tf#L164) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>00-bootstrap</code> | | [prefix](variables.tf#L165) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | <code>00-bootstrap</code> |
| [cicd_repositories](variables.tf#L46) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10; data_platform_dev &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; data_platform_prod &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; networking &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; project_factory_dev &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; project_factory_prod &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; security &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | | | [cicd_repositories](variables.tf#L47) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10; data_platform_dev &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; data_platform_prod &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; networking &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; project_factory_dev &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; project_factory_prod &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; security &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | |
| [custom_roles](variables.tf#L116) | Custom roles defined at the org level, in key => id format. | <code title="object&#40;&#123;&#10; service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | <code>00-bootstrap</code> | | [custom_roles](variables.tf#L117) | Custom roles defined at the org level, in key => id format. | <code title="object&#40;&#123;&#10; service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | <code>00-bootstrap</code> |
| [groups](variables.tf#L125) | Group names to grant organization-level permissions. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; gcp-billing-admins &#61; &#34;gcp-billing-admins&#34;,&#10; gcp-devops &#61; &#34;gcp-devops&#34;,&#10; gcp-network-admins &#61; &#34;gcp-network-admins&#34;&#10; gcp-organization-admins &#61; &#34;gcp-organization-admins&#34;&#10; gcp-security-admins &#61; &#34;gcp-security-admins&#34;&#10; gcp-support &#61; &#34;gcp-support&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | <code>00-bootstrap</code> | | [groups](variables.tf#L126) | Group names to grant organization-level permissions. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; gcp-billing-admins &#61; &#34;gcp-billing-admins&#34;,&#10; gcp-devops &#61; &#34;gcp-devops&#34;,&#10; gcp-network-admins &#61; &#34;gcp-network-admins&#34;&#10; gcp-organization-admins &#61; &#34;gcp-organization-admins&#34;&#10; gcp-security-admins &#61; &#34;gcp-security-admins&#34;&#10; gcp-support &#61; &#34;gcp-support&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | <code>00-bootstrap</code> |
| [organization_policy_configs](variables.tf#L150) | Organization policies customization. | <code title="object&#40;&#123;&#10; allowed_policy_member_domains &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | | | [organization_policy_configs](variables.tf#L151) | Organization policies customization. | <code title="object&#40;&#123;&#10; allowed_policy_member_domains &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | |
| [outputs_location](variables.tf#L158) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> | | <code>null</code> | | | [outputs_location](variables.tf#L159) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> | | <code>null</code> | |
| [tag_names](variables.tf#L175) | Customized names for resource management tags. | <code title="object&#40;&#123;&#10; context &#61; string&#10; environment &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; context &#61; &#34;context&#34;&#10; environment &#61; &#34;environment&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | | | [tag_names](variables.tf#L176) | Customized names for resource management tags. | <code title="object&#40;&#123;&#10; context &#61; string&#10; environment &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; context &#61; &#34;context&#34;&#10; environment &#61; &#34;environment&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [team_folders](variables.tf#L192) | Team folders to be created. Format is described in a code comment. | <code title="map&#40;object&#40;&#123;&#10; descriptive_name &#61; string&#10; group_iam &#61; map&#40;list&#40;string&#41;&#41;&#10; impersonation_groups &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>null</code> | | | [team_folders](variables.tf#L193) | Team folders to be created. Format is described in a code comment. | <code title="map&#40;object&#40;&#123;&#10; descriptive_name &#61; string&#10; group_iam &#61; map&#40;list&#40;string&#41;&#41;&#10; impersonation_groups &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>null</code> | |
## Outputs ## Outputs
| name | description | sensitive | consumers | | name | description | sensitive | consumers |
|---|---|:---:|---| |---|---|:---:|---|
| [cicd_repositories](outputs.tf#L143) | WIF configuration for CI/CD repositories. | | | | [cicd_repositories](outputs.tf#L145) | WIF configuration for CI/CD repositories. | | |
| [dataplatform](outputs.tf#L155) | Data for the Data Platform stage. | | | | [dataplatform](outputs.tf#L159) | Data for the Data Platform stage. | | |
| [networking](outputs.tf#L171) | Data for the networking stage. | | | | [networking](outputs.tf#L175) | Data for the networking stage. | | |
| [project_factories](outputs.tf#L180) | Data for the project factories stage. | | | | [project_factories](outputs.tf#L184) | Data for the project factories stage. | | |
| [providers](outputs.tf#L196) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>03-dataplatform</code> · <code>xx-sandbox</code> · <code>xx-teams</code> | | [providers](outputs.tf#L200) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>03-dataplatform</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
| [sandbox](outputs.tf#L203) | Data for the sandbox stage. | | <code>xx-sandbox</code> | | [sandbox](outputs.tf#L207) | Data for the sandbox stage. | | <code>xx-sandbox</code> |
| [security](outputs.tf#L213) | Data for the networking stage. | | <code>02-security</code> | | [security](outputs.tf#L217) | Data for the networking stage. | | <code>02-security</code> |
| [teams](outputs.tf#L223) | Data for the teams stage. | | | | [teams](outputs.tf#L227) | Data for the teams stage. | | |
| [tfvars](outputs.tf#L236) | Terraform variable files for the following stages. | ✓ | | | [tfvars](outputs.tf#L240) | Terraform variable files for the following stages. | ✓ | |
<!-- END TFDOC --> <!-- END TFDOC -->

14
fast/stages/01-resman/cicd-data-platform.tf
View File

@ -37,7 +37,7 @@ module "branch-dp-dev-cicd-repo" {
included_files = [ included_files = [
"**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml" "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
] ]
service_account = module.branch-dp-dev-sa.iam_email service_account = module.branch-dp-dev-sa-cicd.0.id
substitutions = {} substitutions = {}
template = { template = {
project_id = null project_id = null
@ -47,6 +47,7 @@ module "branch-dp-dev-cicd-repo" {
} }
} }
} }
depends_on = [module.branch-dp-dev-sa-cicd]
} }
module "branch-dp-prod-cicd-repo" { module "branch-dp-prod-cicd-repo" {
@ -68,7 +69,7 @@ module "branch-dp-prod-cicd-repo" {
included_files = [ included_files = [
"**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml" "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
] ]
service_account = module.branch-dp-prod-sa.iam_email service_account = module.branch-dp-prod-sa-cicd.0.id
substitutions = {} substitutions = {}
template = { template = {
project_id = null project_id = null
@ -78,6 +79,7 @@ module "branch-dp-prod-cicd-repo" {
} }
} }
} }
depends_on = [module.branch-dp-prod-sa-cicd]
} }
# SAs used by CI/CD workflows to impersonate automation SAs # SAs used by CI/CD workflows to impersonate automation SAs
@ -96,7 +98,9 @@ module "branch-dp-dev-sa-cicd" {
iam = ( iam = (
each.value.type == "sourcerepo" each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos # used directly from the cloud build trigger for source repos
? {} ? {
"roles/iam.serviceAccountUser" = local.automation_resman_sa
}
# impersonated via workload identity federation for external repos # impersonated via workload identity federation for external repos
: { : {
"roles/iam.workloadIdentityUser" = [ "roles/iam.workloadIdentityUser" = [
@ -135,7 +139,9 @@ module "branch-dp-prod-sa-cicd" {
iam = ( iam = (
each.value.type == "sourcerepo" each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos # used directly from the cloud build trigger for source repos
? {} ? {
"roles/iam.serviceAccountUser" = local.automation_resman_sa
}
# impersonated via workload identity federation for external repos # impersonated via workload identity federation for external repos
: { : {
"roles/iam.workloadIdentityUser" = [ "roles/iam.workloadIdentityUser" = [

7
fast/stages/01-resman/cicd-networking.tf
View File

@ -35,7 +35,7 @@ module "branch-network-cicd-repo" {
fast-02-networking = { fast-02-networking = {
filename = ".cloudbuild/workflow.yaml" filename = ".cloudbuild/workflow.yaml"
included_files = ["**/*tf", ".cloudbuild/workflow.yaml"] included_files = ["**/*tf", ".cloudbuild/workflow.yaml"]
service_account = module.branch-network-sa.id service_account = module.branch-network-sa-cicd.0.id
substitutions = {} substitutions = {}
template = { template = {
project_id = null project_id = null
@ -45,6 +45,7 @@ module "branch-network-cicd-repo" {
} }
} }
} }
depends_on = [module.branch-network-sa-cicd]
} }
# SA used by CI/CD workflows to impersonate automation SAs # SA used by CI/CD workflows to impersonate automation SAs
@ -63,7 +64,9 @@ module "branch-network-sa-cicd" {
iam = ( iam = (
each.value.type == "sourcerepo" each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos # used directly from the cloud build trigger for source repos
? {} ? {
"roles/iam.serviceAccountUser" = local.automation_resman_sa
}
# impersonated via workload identity federation for external repos # impersonated via workload identity federation for external repos
: { : {
"roles/iam.workloadIdentityUser" = [ "roles/iam.workloadIdentityUser" = [

7
fast/stages/01-resman/cicd-security.tf
View File

@ -35,7 +35,7 @@ module "branch-security-cicd-repo" {
fast-02-security = { fast-02-security = {
filename = ".cloudbuild/workflow.yaml" filename = ".cloudbuild/workflow.yaml"
included_files = ["**/*tf", ".cloudbuild/workflow.yaml"] included_files = ["**/*tf", ".cloudbuild/workflow.yaml"]
service_account = module.branch-security-sa.id service_account = module.branch-security-sa-cicd.0.id
substitutions = {} substitutions = {}
template = { template = {
project_id = null project_id = null
@ -45,6 +45,7 @@ module "branch-security-cicd-repo" {
} }
} }
} }
depends_on = [module.branch-security-sa-cicd]
} }
# SA used by CI/CD workflows to impersonate automation SAs # SA used by CI/CD workflows to impersonate automation SAs
@ -63,7 +64,9 @@ module "branch-security-sa-cicd" {
iam = ( iam = (
each.value.type == "sourcerepo" each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos # used directly from the cloud build trigger for source repos
? {} ? {
"roles/iam.serviceAccountUser" = local.automation_resman_sa
}
# impersonated via workload identity federation for external repos # impersonated via workload identity federation for external repos
: { : {
"roles/iam.workloadIdentityUser" = [ "roles/iam.workloadIdentityUser" = [

14
fast/stages/01-resman/cicd-teams.tf
View File

@ -37,7 +37,7 @@ module "branch-teams-dev-pf-cicd-repo" {
included_files = [ included_files = [
"**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml" "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
] ]
service_account = module.branch-teams-dev-pf-sa.iam_email service_account = module.branch-teams-dev-pf-sa-cicd.0.id
substitutions = {} substitutions = {}
template = { template = {
project_id = null project_id = null
@ -47,6 +47,7 @@ module "branch-teams-dev-pf-cicd-repo" {
} }
} }
} }
depends_on = [module.branch-teams-dev-pf-sa-cicd]
} }
module "branch-teams-prod-pf-cicd-repo" { module "branch-teams-prod-pf-cicd-repo" {
@ -68,7 +69,7 @@ module "branch-teams-prod-pf-cicd-repo" {
included_files = [ included_files = [
"**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml" "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
] ]
service_account = module.branch-teams-prod-pf-sa.iam_email service_account = module.branch-teams-prod-pf-sa-cicd.0.id
substitutions = {} substitutions = {}
template = { template = {
project_id = null project_id = null
@ -78,6 +79,7 @@ module "branch-teams-prod-pf-cicd-repo" {
} }
} }
} }
depends_on = [module.branch-teams-prod-pf-sa-cicd]
} }
# SAs used by CI/CD workflows to impersonate automation SAs # SAs used by CI/CD workflows to impersonate automation SAs
@ -96,7 +98,9 @@ module "branch-teams-dev-pf-sa-cicd" {
iam = ( iam = (
each.value.type == "sourcerepo" each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos # used directly from the cloud build trigger for source repos
? {} ? {
"roles/iam.serviceAccountUser" = local.automation_resman_sa
}
# impersonated via workload identity federation for external repos # impersonated via workload identity federation for external repos
: { : {
"roles/iam.workloadIdentityUser" = [ "roles/iam.workloadIdentityUser" = [
@ -135,7 +139,9 @@ module "branch-teams-prod-pf-sa-cicd" {
iam = ( iam = (
each.value.type == "sourcerepo" each.value.type == "sourcerepo"
# used directly from the cloud build trigger for source repos # used directly from the cloud build trigger for source repos
? {} ? {
"roles/iam.serviceAccountUser" = local.automation_resman_sa
}
# impersonated via workload identity federation for external repos # impersonated via workload identity federation for external repos
: { : {
"roles/iam.workloadIdentityUser" = [ "roles/iam.workloadIdentityUser" = [

11
fast/stages/01-resman/main.tf
View File

@ -16,6 +16,13 @@
locals { locals {
# convenience flags that express where billing account resides # convenience flags that express where billing account resides
automation_resman_sa = try(
[format(
"serviceAccount:%s",
data.google_client_openid_userinfo.provider_identity.0.email
)],
[]
)
billing_ext = var.billing_account.organization_id == null billing_ext = var.billing_account.organization_id == null
billing_org = var.billing_account.organization_id == var.organization.id billing_org = var.billing_account.organization_id == var.organization.id
billing_org_ext = !local.billing_ext && !local.billing_org billing_org_ext = !local.billing_ext && !local.billing_org
@ -64,3 +71,7 @@ locals {
try(var.automation.federated_identity_providers, null), {} try(var.automation.federated_identity_providers, null), {}
) )
} }
data "google_client_openid_userinfo" "provider_identity" {
count = length(local.cicd_repositories) > 0 ? 1 : 0
}

8
fast/stages/01-resman/outputs.tf
View File

@ -52,7 +52,9 @@ locals {
for k, v in local.cicd_repositories : k => templatefile( for k, v in local.cicd_repositories : k => templatefile(
"${path.module}/templates/workflow-${v.type}.yaml", "${path.module}/templates/workflow-${v.type}.yaml",
merge(local.cicd_workflow_attrs[k], { merge(local.cicd_workflow_attrs[k], {
identity_provider = local.identity_providers[v.identity_provider].name identity_provider = try(
local.identity_providers[v.identity_provider].name, null
)
outputs_bucket = var.automation.outputs_bucket outputs_bucket = var.automation.outputs_bucket
stage_name = k stage_name = k
}) })
@ -146,7 +148,9 @@ output "cicd_repositories" {
for k, v in local.cicd_repositories : k => { for k, v in local.cicd_repositories : k => {
branch = v.branch branch = v.branch
name = v.name name = v.name
provider = local.identity_providers[v.identity_provider].name provider = try(
local.identity_providers[v.identity_provider].name, null
)
service_account = local.cicd_workflow_attrs[k].service_account service_account = local.cicd_workflow_attrs[k].service_account
} if v != null } if v != null
} }

1
fast/stages/01-resman/variables.tf
View File

@ -23,6 +23,7 @@ variable "automation" {
type = object({ type = object({
outputs_bucket = string outputs_bucket = string
project_id = string project_id = string
project_number = string
federated_identity_pool = string federated_identity_pool = string
federated_identity_providers = map(object({ federated_identity_providers = map(object({
issuer = string issuer = string

2
fast/stages/03-project-factory/dev/main.tf
View File

@ -50,7 +50,7 @@ module "projects" {
prefix = var.prefix prefix = var.prefix
service_accounts = try(each.value.service_accounts, {}) service_accounts = try(each.value.service_accounts, {})
services = try(each.value.services, []) services = try(each.value.services, [])
service_identities_iam = try(each.value.services_iam, {}) service_identities_iam = try(each.value.service_identities_iam, {})
vpc = try(each.value.vpc, null) vpc = try(each.value.vpc, null)
} }

110
fast/stages/CLEANUP.md Normal file
View File

@ -0,0 +1,110 @@
# FAST deployment clean up
In case you require destroying a previous FAST deployment in your organization, follow these steps.
Destruction must be done in reverse order, from stage 3 to stage 0:
## Stage 3 (Project Factory)
```bash
cd $FAST_PWD/03-project-factory/prod/
terraform destroy
```
## Stage 3 (GKE)
Terraform refuses to delete non-empty GCS buckets and/or BigQuery datasets, so they need to be removed manually from tf state
```bash
cd $FAST_PWD/03-project-factory/prod/
# remove BQ dataset manually
for x in $(terraform state list | grep google_bigquery_dataset); do
terraform state rm "$x";
done
terraform destroy
```
## Stage 2 (Security)
```bash
cd $FAST_PWD/02-security/
terraform destroy
```
## Stage 2 (Networking)
```bash
cd $FAST_PWD/02-networking-XXX/
terraform destroy
```
There's a minor glitch that can surface running terraform destroy, where the service project attachments to the Shared VPC will not get destroyed even with the relevant API call succeeding. We are investigating the issue, in the meantime just manually remove the attachment in the Cloud console or via the ```gcloud beta compute shared-vpc associated-projects remove``` [command](https://cloud.google.com/sdk/gcloud/reference/beta/compute/shared-vpc/associated-projects/remove) when terraform destroy fails, and then relaunch the command.
## Stage 1 (Resource Management)
Stage 1 is a little more complicated because of the GCS Buckets. By default terraform refuses to delete non-empty buckets, which is a good thing for your terraform state. However, it makes destruction a bit harder
```bash
cd $FAST_PWD/01-resman/
# remove buckets from state since terraform refuses to delete them
for x in $(terraform state list | grep google_storage_bucket.bucket); do
terraform state rm "$x"
done
terraform destroy
```
## Stage 0 (Bootstrap)
**You should follow these steps carefully because we can end up destroying our own permissions. As we will be removing gcp-admins group roles, where your user belongs to, you will be required to grant organization admin role again**
We also have to remove several resources (GCS buckets and BQ datasets) manually.
```bash
cd $FAST_PWD/00-bootstrap/
# remove provider config to execute without SA impersonation
rm 00-bootstrap-providers.tf
# migrate to local state
terraform init -migrate-state
# remove GCS buckets and BQ dataset manually
for x in $(terraform state list | grep google_storage_bucket.bucket); do
terraform state rm "$x";
done
for x in $(terraform state list | grep google_bigquery_dataset); do
terraform state rm "$x";
done
terraform destroy
# when this fails continue with the steps below
# make your user (the one you are using to execute this step) org admin again, as we will remove organization-admins group roles
# Add the Organization Admin role to $BU_USER in the GCP Console
# grant yourself this permission so you can finish the destruction
export FAST_DESTROY_ROLES="roles/billing.admin roles/logging.admin \
roles/iam.organizationRoleAdmin roles/resourcemanager.projectDeleter \
roles/resourcemanager.folderAdmin roles/owner"
export FAST_BU=$(gcloud config list --format 'value(core.account)')
# find your org id
gcloud organizations list --filter display_name:[part of your domain]
# set your org id
export FAST_ORG_ID=XXXX
for role in $FAST_DESTROY_ROLES; do
gcloud organizations add-iam-policy-binding $FAST_ORG_ID \
--member user:$FAST_BU --role $role
done
terraform destroy
rm -i terraform.tfstate*
```
In case you are willing to deploy FAST stages again, the following changes shall be done before:
* Modify the [prefix](00-bootstrap/variables.tf) variable to allow the deployment of resources that need unique names (eg, projects).
* Modify the [custom_roles](00-bootstrap/variables.tf) variable to allow recently deleted custom roles to be created again.

2
fast/stages/README.md
View File

@ -17,6 +17,8 @@ To achieve this, we rely on specific GCP functionality like [delegated role gran
Refer to each stage's documentation for a detailed description of its purpose, the architectural choices made in its design, and how it can be configured and wired together to terraform a whole GCP organization. The following is a brief overview of each stage. Refer to each stage's documentation for a detailed description of its purpose, the architectural choices made in its design, and how it can be configured and wired together to terraform a whole GCP organization. The following is a brief overview of each stage.
To destroy a previous FAST deployment follow the instructions detailed in [cleanup](CLEANUP.md).
## Organizational level (00-01) ## Organizational level (00-01)
- [Bootstrap](00-bootstrap/README.md) - [Bootstrap](00-bootstrap/README.md)

11
modules/cloud-function/README.md
View File

@ -173,11 +173,12 @@ module "cf-http" {
| [labels](variables.tf#L82) | Resource labels. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> | | [labels](variables.tf#L82) | Resource labels. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> |
| [prefix](variables.tf#L93) | Optional prefix used for resource names. | <code>string</code> | | <code>null</code> | | [prefix](variables.tf#L93) | Optional prefix used for resource names. | <code>string</code> | | <code>null</code> |
| [region](variables.tf#L104) | Region used for all resources. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> | | [region](variables.tf#L104) | Region used for all resources. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| [service_account](variables.tf#L110) | Service account email. Unused if service account is auto-created. | <code>string</code> | | <code>null</code> | | [secrets](variables.tf#L110) | Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format. | <code title="map&#40;object&#40;&#123;&#10; is_volume &#61; bool&#10; project_id &#61; number&#10; secret &#61; string&#10; versions &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [service_account_create](variables.tf#L116) | Auto-create service account. | <code>bool</code> | | <code>false</code> | | [service_account](variables.tf#L122) | Service account email. Unused if service account is auto-created. | <code>string</code> | | <code>null</code> |
| [trigger_config](variables.tf#L122) | Function trigger configuration. Leave null for HTTP trigger. | <code title="object&#40;&#123;&#10; event &#61; string&#10; resource &#61; string&#10; retry &#61; bool&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | | [service_account_create](variables.tf#L128) | Auto-create service account. | <code>bool</code> | | <code>false</code> |
| [vpc_connector](variables.tf#L132) | VPC connector configuration. Set create to 'true' if a new connector needs to be created. | <code title="object&#40;&#123;&#10; create &#61; bool&#10; name &#61; string&#10; egress_settings &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | | [trigger_config](variables.tf#L134) | Function trigger configuration. Leave null for HTTP trigger. | <code title="object&#40;&#123;&#10; event &#61; string&#10; resource &#61; string&#10; retry &#61; bool&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [vpc_connector_config](variables.tf#L142) | VPC connector network configuration. Must be provided if new VPC connector is being created. | <code title="object&#40;&#123;&#10; ip_cidr_range &#61; string&#10; network &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | | [vpc_connector](variables.tf#L144) | VPC connector configuration. Set create to 'true' if a new connector needs to be created. | <code title="object&#40;&#123;&#10; create &#61; bool&#10; name &#61; string&#10; egress_settings &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [vpc_connector_config](variables.tf#L154) | VPC connector network configuration. Must be provided if new VPC connector is being created. | <code title="object&#40;&#123;&#10; ip_cidr_range &#61; string&#10; network &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
## Outputs ## Outputs

29
modules/cloud-function/main.tf
View File

@ -91,6 +91,35 @@ resource "google_cloudfunctions_function" "function" {
} }
} }
dynamic "secret_environment_variables" {
for_each = { for k, v in var.secrets : k => v if !v.is_volume }
iterator = secret
content {
key = secret.key
project_id = secret.value.project_id
secret = secret.value.secret
version = try(secret.value.versions.0, "latest")
}
}
dynamic "secret_volumes" {
for_each = { for k, v in var.secrets : k => v if v.is_volume }
iterator = secret
content {
mount_path = secret.key
project_id = secret.value.project_id
secret = secret.value.secret
dynamic "versions" {
for_each = secret.value.versions
iterator = version
content {
path = split(":", version)[1]
version = split(":", version)[0]
}
}
}
}
} }
resource "google_cloudfunctions_function_iam_binding" "default" { resource "google_cloudfunctions_function_iam_binding" "default" {

12
modules/cloud-function/variables.tf
View File

@ -107,6 +107,18 @@ variable "region" {
default = "europe-west1" default = "europe-west1"
} }
variable "secrets" {
description = "Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format."
type = map(object({
is_volume = bool
project_id = number
secret = string
versions = list(string)
}))
nullable = false
default = {}
}
variable "service_account" { variable "service_account" {
description = "Service account email. Unused if service account is auto-created." description = "Service account email. Unused if service account is auto-created."
type = string type = string

10
modules/iam-service-account/README.md
View File

@ -2,6 +2,8 @@
This module allows simplified creation and management of one a service account and its IAM bindings. A key can optionally be generated and will be stored in Terraform state. To use it create a sensitive output in your root modules referencing the `key` output, then extract the private key from the JSON formatted outputs. Alternatively, the `key` can be generated with `openssl` library and only public part uploaded to the Service Account, for more refer to the [Onprem SA Key Management](../../examples/cloud-operations/onprem-sa-key-management/) example. This module allows simplified creation and management of one a service account and its IAM bindings. A key can optionally be generated and will be stored in Terraform state. To use it create a sensitive output in your root modules referencing the `key` output, then extract the private key from the JSON formatted outputs. Alternatively, the `key` can be generated with `openssl` library and only public part uploaded to the Service Account, for more refer to the [Onprem SA Key Management](../../examples/cloud-operations/onprem-sa-key-management/) example.
Note that this module does not fully comply with our design principles, as outputs have no dependencies on IAM bindings to prevent resource cycles.
## Example ## Example
```hcl ```hcl
@ -64,9 +66,9 @@ module "myproject-default-service-accounts" {
| [email](outputs.tf#L17) | Service account email. | | | [email](outputs.tf#L17) | Service account email. | |
| [iam_email](outputs.tf#L25) | IAM-format service account email. | | | [iam_email](outputs.tf#L25) | IAM-format service account email. | |
| [id](outputs.tf#L33) | Service account id. | | | [id](outputs.tf#L33) | Service account id. | |
| [key](outputs.tf#L38) | Service account key. | ✓ | | [key](outputs.tf#L41) | Service account key. | ✓ |
| [name](outputs.tf#L44) | Service account name. | | | [name](outputs.tf#L47) | Service account name. | |
| [service_account](outputs.tf#L49) | Service account resource. | | | [service_account](outputs.tf#L52) | Service account resource. | |
| [service_account_credentials](outputs.tf#L54) | Service account json credential templates for uploaded public keys data. | | | [service_account_credentials](outputs.tf#L57) | Service account json credential templates for uploaded public keys data. | |
<!-- END TFDOC --> <!-- END TFDOC -->

3
modules/iam-service-account/outputs.tf
View File

@ -33,6 +33,9 @@ output "iam_email" {
output "id" { output "id" {
description = "Service account id." description = "Service account id."
value = local.service_account.id value = local.service_account.id
depends_on = [
local.service_account
]
} }
output "key" { output "key" {

1
tests/fast/stages/s01_resman/fixture/main.tf
View File

@ -20,6 +20,7 @@ module "stage" {
federated_identity_pool = null federated_identity_pool = null
federated_identity_providers = null federated_identity_providers = null
project_id = "fast-prod-automation" project_id = "fast-prod-automation"
project_number = 123456
outputs_bucket = "test" outputs_bucket = "test"
} }
billing_account = { billing_account = {

18
tools/check_names.py
View File

@ -84,13 +84,21 @@ def main(dirs, prefix_length=None):
source_just = max(len(k) for k in MOD_LIMITS) source_just = max(len(k) for k in MOD_LIMITS)
name_just = max(len(n.name) for n in names) name_just = max(len(n.name) for n in names)
value_just = max(len(n.value) for n in names) value_just = max(len(n.value) for n in names)
errors = []
for name in names: for name in names:
name_length = name.length + prefix_length name_length = name.length + prefix_length
flag = '✗' if name_length >= MOD_LIMITS[name.source] else '✓' if name_length >= MOD_LIMITS[name.source]:
print(f'[{flag}] {name.source.ljust(source_just)} ' flag = "✗"
f'{name.name.ljust(name_just)} ' errors += [f"{name.source}:{name.name}:{name_length}"]
f'{name.value.ljust(value_just)} ' else:
f'({name_length})') flag = "✓"
print(f"[{flag}] {name.source.ljust(source_just)} "
f"{name.name.ljust(name_just)} "
f"{name.value.ljust(value_just)} "
f"({name_length})")
if errors:
raise ValueError(errors)
if __name__ == '__main__': if __name__ == '__main__':